mirror/SDRPlusPlus
0
0
Fork 0
mirror of https://github.com/AlexandreRouma/SDRPlusPlus.git synced 2025-01-23 08:24:44 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #616 from AlexandreRouma/zstd_compression

Browse Source 
Optional ZSTD compression in SDR++ server
This commit is contained in:
AlexandreRouma 2022-01-27 17:29:32 +01:00 committed by GitHub
commit 97346e6621
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 91 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build_all.yml vendored
View File

@ -51,7 +51,7 @@ jobs:
run: mkdir "C:/Program Files/codec2" ; mkdir "C:/Program Files/codec2/include" ; mkdir "C:/Program Files/codec2/include/codec2" ; mkdir "C:/Program Files/codec2/lib" ; cd "codec2" ; xcopy "src" "C:/Program Files/codec2/include" ; cd "build" ; xcopy "src" "C:/Program Files/codec2/lib" ; xcopy "codec2" "C:/Program Files/codec2/include/codec2"
- name: Install vcpkg dependencies
run: vcpkg install fftw3:x64-windows glfw3:x64-windows portaudio:x64-windows
run: vcpkg install fftw3:x64-windows glfw3:x64-windows portaudio:x64-windows zstd:x64-windows
- name: Install rtaudio
run: git clone https://github.com/thestk/rtaudio ; cd rtaudio ; git checkout 2f2fca4502d506abc50f6d4473b2836d24cfb1e3 ; mkdir build ; cd build ; cmake .. ; cmake --build . --config Release ; cmake --install .
@ -87,7 +87,7 @@ jobs:
run: brew update
- name: Install dependencies
run: brew install fftw glfw airspy airspyhf portaudio hackrf rtl-sdr libbladerf codec2 && pip3 install mako
run: brew install fftw glfw airspy airspyhf portaudio hackrf rtl-sdr libbladerf codec2 && pip3 install mako zstd
- name: Install volk
run: git clone --recursive https://github.com/gnuradio/volk && cd volk && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release .. && make -j3 && sudo make install && cd ../../

8
core/CMakeLists.txt
View File

@ -71,6 +71,10 @@ if (MSVC)
# WinSock2
target_link_libraries(sdrpp_core PUBLIC wsock32 ws2_32)
# ZSTD
find_package(zstd CONFIG REQUIRED)
target_link_libraries(sdrpp_core PUBLIC zstd::libzstd_shared)
else()
find_package(PkgConfig)
find_package(OpenGL REQUIRED)
@ -78,12 +82,14 @@ else()
pkg_check_modules(FFTW3 REQUIRED fftw3f)
pkg_check_modules(VOLK REQUIRED volk)
pkg_check_modules(GLFW3 REQUIRED glfw3)
pkg_check_modules(LIBZSTD REQUIRED libzstd)
target_include_directories(sdrpp_core PUBLIC
${OPENGL_INCLUDE_DIRS}
${FFTW3_INCLUDE_DIRS}
${GLFW3_INCLUDE_DIRS}
${VOLK_INCLUDE_DIRS}
${LIBZSTD_INCLUDE_DIRS}
)
target_link_directories(sdrpp_core PUBLIC
@ -91,6 +97,7 @@ else()
${FFTW3_LIBRARY_DIRS}
${GLFW3_LIBRARY_DIRS}
${VOLK_LIBRARY_DIRS}
${LIBZSTD_LIBRARY_DIRS}
)
target_link_libraries(sdrpp_core PUBLIC
@ -98,6 +105,7 @@ else()
${FFTW3_LIBRARIES}
${GLFW3_LIBRARIES}
${VOLK_LIBRARIES}
${LIBZSTD_LIBRARIES}
)
if (NOT USE_INTERNAL_LIBCORRECT)

7
core/src/dsp/compression.h
View File

@ -12,11 +12,16 @@ namespace dsp {
public:
DynamicRangeCompressor() {}
DynamicRangeCompressor(stream<complex_t>* in, PCMType pcmType) { init(in, pcmType); }
DynamicRangeCompressor(stream<complex_t>* in, PCMType pcmType) {
init(in, pcmType);
}
void init(stream<complex_t>* in, PCMType pcmType) {
_in = in;
_pcmType = pcmType;
out.setBufferSize((sizeof(dsp::complex_t) * STREAM_BUFFER_SIZE) + 8);
generic_block<DynamicRangeCompressor>::registerInput(_in);
generic_block<DynamicRangeCompressor>::registerOutput(&out);
generic_block<DynamicRangeCompressor>::_block_init = true;

7
core/src/dsp/stream.h
View File

@ -31,6 +31,13 @@ namespace dsp {
volk_free(readBuf);
}
void setBufferSize(int samples) {
volk_free(writeBuf);
volk_free(readBuf);
writeBuf = (T*)volk_malloc(samples * sizeof(T), volk_get_alignment());
readBuf = (T*)volk_malloc(samples * sizeof(T), volk_get_alignment());
}
bool swap(int size) {
{
// Wait to either swap or stop

30
core/src/server.cpp
View File

@ -10,6 +10,7 @@
#include <gui/smgui.h>
#include <utils/optionlist.h>
#include <dsp/compression.h>
#include <zstd.h>
namespace server {
dsp::stream<dsp::complex_t> dummyInput;
@ -35,11 +36,14 @@ namespace server {
SmGui::DrawListElem dummyElem;
ZSTD_CCtx* cctx;
net::Listener listener;
OptionList<std::string, std::string> sourceList;
int sourceId = 0;
bool running = false;
bool compression = false;
double sampleRate = 1000000.0;
int main() {
@ -68,9 +72,8 @@ namespace server {
bb_pkt_hdr = (PacketHeader*)bbuf;
bb_pkt_data = &bbuf[sizeof(PacketHeader)];
// Terminate config manager
core::configManager.disableAutoSave();
core::configManager.save();
// Initialize compressor
cctx = ZSTD_createCCtx();
core::configManager.acquire();
std::string modulesDir = core::configManager.conf["modulesDirectory"];
@ -183,6 +186,7 @@ namespace server {
// Perform settings reset
sigpath::sourceManager.stop();
comp.setPCMType(dsp::PCM_TYPE_I16);
compression = false;
sendSampleRate(sampleRate);
@ -218,14 +222,19 @@ namespace server {
}
void _testServerHandler(uint8_t* data, int count, void* ctx) {
// Build data packet
PacketHeader* hdr = (PacketHeader*)bbuf;
hdr->type = PACKET_TYPE_BASEBAND;
hdr->size = sizeof(PacketHeader) + count;
memcpy(&bbuf[sizeof(PacketHeader)], data, count);
// Compress data if needed and fill out header fields
if (compression) {
bb_pkt_hdr->type = PACKET_TYPE_BASEBAND_COMPRESSED;
bb_pkt_hdr->size = sizeof(PacketHeader) + (uint32_t)ZSTD_compressCCtx(cctx, &bbuf[sizeof(PacketHeader)], SERVER_MAX_PACKET_SIZE, data, count, 1);
}
else {
bb_pkt_hdr->type = PACKET_TYPE_BASEBAND;
bb_pkt_hdr->size = sizeof(PacketHeader) + count;
memcpy(&bbuf[sizeof(PacketHeader)], data, count);
}
// Write to network
if (client && client->isOpen()) { client->write(hdr->size, bbuf); }
if (client && client->isOpen()) { client->write(bb_pkt_hdr->size, bbuf); }
}
void setInput(dsp::stream<dsp::complex_t>* stream) {
@ -281,6 +290,9 @@ namespace server {
dsp::PCMType type = (dsp::PCMType)*(uint8_t*)data;
comp.setPCMType(type);
}
else if (cmd == COMMAND_SET_COMPRESSION && len == 1) {
compression = *(uint8_t*)data;
}
else {
spdlog::error("Invalid Command: {0} (len = {1})", cmd, len);
sendError(ERROR_INVALID_COMMAND);

2
core/src/server_protocol.h
View File

@ -11,6 +11,7 @@ namespace server {
PACKET_TYPE_COMMAND,
PACKET_TYPE_COMMAND_ACK,
PACKET_TYPE_BASEBAND,
PACKET_TYPE_BASEBAND_COMPRESSED,
PACKET_TYPE_VFO,
PACKET_TYPE_FFT,
PACKET_TYPE_ERROR
@ -25,6 +26,7 @@ namespace server {
COMMAND_SET_FREQUENCY,
COMMAND_GET_SAMPLERATE,
COMMAND_SET_SAMPLE_TYPE,
COMMAND_SET_COMPRESSION,
// Server to client
COMMAND_SET_SAMPLERATE = 0x80,

2
docker_builds/debian_bullseye/do_build.sh
View File

@ -4,7 +4,7 @@ cd /root
# Install dependencies and tools
apt update
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
libcodec2-dev

2
docker_builds/debian_buster/do_build.sh
View File

@ -4,7 +4,7 @@ cd /root
# Install dependencies and tools
apt update
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
libcodec2-dev

2
docker_builds/debian_sid/do_build.sh
View File

@ -4,7 +4,7 @@ cd /root
# Install dependencies and tools
apt update
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
libcodec2-dev

4
docker_builds/raspios_bullseye/Dockerfile
View File

@ -1,4 +0,0 @@
FROM ubuntu:latest
ENV DEBIAN_FRONTEND=noninteractive
COPY do_build.sh /root
RUN chmod +x /root/do_build.sh

48
docker_builds/raspios_bullseye/do_build.sh
View File

@ -1,48 +0,0 @@
#!/bin/bash
set -e
cd /root
# Install tools
apt update
apt install -y wget p7zip-full qemu-user-static gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf build-essential cmake pkg-config
# Download and extract raspberry pi image
wget https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-11-08/2021-10-30-raspios-bullseye-armhf-lite.zip
7z x 2021-10-30-raspios-bullseye-armhf-lite.zip
7z x 2021-10-30-raspios-bullseye-armhf-lite.img
# Expand and mount rootfs image
dd if=/dev/zero bs=1G count=1 >> 1.img
e2fsck -p -f 1.img
resize2fs 1.img
mount 1.img /mnt
# Copy qemu to the rootfs
cp /usr/bin/qemu-arm-static /mnt/bin/
# Inject script to install dependencies
echo 'export DEBIAN_FRONTEND=noninteractive' >> /mnt/root/prepare.sh
echo 'apt update --allow-releaseinfo-change' >> /mnt/root/prepare.sh
echo 'apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \' >> /mnt/root/prepare.sh
echo ' libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \' >> /mnt/root/prepare.sh
echo ' libcodec2-dev' >> /mnt/root/prepare.sh
# Run prepare.sh script
chroot /mnt /bin/qemu-arm-static /bin/bash /root/prepare.sh
# Setup environment variables
export PKG_CONFIG_PATH=''
export PKG_CONFIG_LIBDIR='/mnt/usr/lib/arm-linux-gnueabihf/pkgconfig:/mnt/usr/lib/pkgconfig:/mnt/usr/share/pkgconfig'
export PKG_CONFIG_SYSROOT_DIR='/mnt'
# Build SDR++
cd SDRPlusPlus
mkdir build
cd build
cmake .. -DCMAKE_TOOLCHAIN_FILE=../docker_builds/raspios_bullseye/rpi_toolchain.cmake -DOPT_BUILD_BLADERF_SOURCE=ON -DOPT_BUILD_LIMESDR_SOURCE=ON -DOPT_BUILD_NEW_PORTAUDIO_SINK=ON -DOPT_BUILD_M17_DECODER=ON
make VERBOSE=1 -j2
# Create deb
cd ..
sh make_debian_package.sh ./build 'libfftw3-dev, libglfw3-dev, libvolk2-dev, librtaudio-dev'
mv sdrpp_debian_amd64.deb sdrpp_raspios_arm32.deb

12
docker_builds/raspios_bullseye/rpi_toolchain.cmake
View File

@ -1,12 +0,0 @@
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR arm)
set(CMAKE_SYSROOT /mnt)
set(CMAKE_C_COMPILER /usr/bin/arm-linux-gnueabihf-gcc)
set(CMAKE_CXX_COMPILER /usr/bin/arm-linux-gnueabihf-g++)
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

2
docker_builds/ubuntu_bionic/do_build.sh
View File

@ -10,7 +10,7 @@ echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://ap
apt update
# Install dependencies and tools
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libsoapysdr-dev libairspy-dev \
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libzstd-dev libsoapysdr-dev libairspy-dev \
libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
libcodec2-dev

2
docker_builds/ubuntu_focal/do_build.sh
View File

@ -4,7 +4,7 @@ cd /root
# Install dependencies and tools
apt update
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
libcodec2-dev

2
docker_builds/ubuntu_hirsute/do_build.sh
View File

@ -4,7 +4,7 @@ cd /root
# Install dependencies and tools
apt update
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
libcodec2-dev

2
docker_builds/ubuntu_impish/do_build.sh
View File

@ -4,7 +4,7 @@ cd /root
# Install dependencies and tools
apt update
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
libcodec2-dev

5
readme.md
View File

@ -78,7 +78,8 @@ brew install \
portaudio \
rtl-sdr \
soapyrtlsdr \
volk
volk \
zstd
mkdir build
cd build
cmake .. \
@ -115,6 +116,7 @@ After this, install the following dependencies using vcpkg:
* fftw3
* glfw3
* zstd
You are probably going to build in 64 bit so make sure vcpkg installs the correct versions using `.\vcpkg.exe install <package>:x64-windows`
@ -221,6 +223,7 @@ you can disable it using the module parameter listed in the table below
* fftw3
* glfw
* libvolk
* zstd
Next install dependencies based on the modules you wish to build (See previous step)

2
rpi_install.sh
View File

@ -5,7 +5,7 @@ set -e
echo "Installing dependencies"
sudo apt update
sudo apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
sudo apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget
echo "Preparing build"

18
source_modules/sdrpp_server_source/src/main.cpp
View File

@ -194,11 +194,18 @@ private:
config.conf["servers"][_this->devConfName]["sampleType"] = _this->sampleTypeList.key(_this->sampleTypeId);
config.release(true);
}
if (ImGui::Checkbox("Compression", &_this->compression)) {
_this->client->setCompression(_this->compression);
bool dummy = false;
// Save config
config.acquire();
config.conf["servers"][_this->devConfName]["compression"] = _this->compression;
config.release(true);
}
bool dummy = true;
style::beginDisabled();
ImGui::Checkbox("Compression", &dummy);
dummy = true;
ImGui::Checkbox("Full IQ", &dummy);
style::endDisabled();
@ -237,9 +244,13 @@ private:
std::string key = config.conf["servers"][devConfName]["sampleType"];
if (sampleTypeList.keyExists(key)) { sampleTypeId = sampleTypeList.keyId(key); }
}
if (config.conf["servers"][devConfName].contains("compression")) {
compression = config.conf["servers"][devConfName]["compression"];
}
// Set settings
client->setSampleType(sampleTypeList[sampleTypeId]);
client->setCompression(compression);
}
std::string name;
@ -261,6 +272,7 @@ private:
OptionList<std::string, dsp::PCMType> sampleTypeList;
int sampleTypeId;
bool compression = false;
server::Client client;
};

14
source_modules/sdrpp_server_source/src/sdrpp_server_client.cpp
View File

@ -26,7 +26,11 @@ namespace server {
s_cmd_hdr = (CommandHeader*)s_pkt_data;
s_cmd_data = &sbuffer[sizeof(PacketHeader) + sizeof(CommandHeader)];
// Initialize decompressor
dctx = ZSTD_createDCtx();
// Initialize DSP
decompIn.setBufferSize((sizeof(dsp::complex_t) * STREAM_BUFFER_SIZE) + 8);
decomp.init(&decompIn);
link.init(&decomp.out, output);
decomp.start();
@ -43,6 +47,7 @@ namespace server {
ClientClass::~ClientClass() {
close();
ZSTD_freeDCtx(dctx);
delete[] rbuffer;
delete[] sbuffer;
}
@ -108,6 +113,11 @@ namespace server {
sendCommand(COMMAND_SET_SAMPLE_TYPE, 1);
}
void ClientClass::setCompression(bool enabled) {
s_cmd_data[0] = enabled;
sendCommand(COMMAND_SET_COMPRESSION, 1);
}
void ClientClass::start() {
if (!client || !client->isOpen()) { return; }
sendCommand(COMMAND_START, 0);
@ -189,6 +199,10 @@ namespace server {
memcpy(_this->decompIn.writeBuf, &buf[sizeof(PacketHeader)], _this->r_pkt_hdr->size - sizeof(PacketHeader));
_this->decompIn.swap(_this->r_pkt_hdr->size - sizeof(PacketHeader));
}
else if (_this->r_pkt_hdr->type == PACKET_TYPE_BASEBAND_COMPRESSED) {
size_t outCount = ZSTD_decompressDCtx(_this->dctx, _this->decompIn.writeBuf, STREAM_BUFFER_SIZE, _this->r_pkt_data, _this->r_pkt_hdr->size - sizeof(PacketHeader));
if (outCount) { _this->decompIn.swap(outCount); };
}
else if (_this->r_pkt_hdr->type == PACKET_TYPE_ERROR) {
spdlog::error("SDR++ Server Error: {0}", buf[sizeof(PacketHeader)]);
}

4
source_modules/sdrpp_server_source/src/sdrpp_server_client.h
View File

@ -11,6 +11,7 @@
#include <dsp/compression.h>
#include <dsp/sink.h>
#include <dsp/link.h>
#include <zstd.h>
#define RFSPACE_MAX_SIZE 8192
#define RFSPACE_HEARTBEAT_INTERVAL_MS 1000
@ -85,6 +86,7 @@ namespace server {
double getSampleRate();
void setSampleType(dsp::PCMType type);
void setCompression(bool enabled);
void start();
void stop();
@ -133,6 +135,8 @@ namespace server {
SmGui::DrawList dl;
std::mutex dlMtx;
ZSTD_DCtx* dctx;
double currentSampleRate = 1000000.0;
};