Merge pull request #616 from AlexandreRouma/zstd_compression

Optional ZSTD compression in SDR++ server

.github/workflows/build_all.yml

@@ -51,7 +51,7 @@ jobs:
           run: mkdir "C:/Program Files/codec2" ; mkdir "C:/Program Files/codec2/include" ; mkdir "C:/Program Files/codec2/include/codec2" ; mkdir "C:/Program Files/codec2/lib" ; cd "codec2" ; xcopy "src" "C:/Program Files/codec2/include" ; cd "build" ; xcopy "src" "C:/Program Files/codec2/lib" ; xcopy "codec2" "C:/Program Files/codec2/include/codec2"
 
         - name: Install vcpkg dependencies
-          run: vcpkg install fftw3:x64-windows glfw3:x64-windows portaudio:x64-windows
+          run: vcpkg install fftw3:x64-windows glfw3:x64-windows portaudio:x64-windows zstd:x64-windows
 
         - name: Install rtaudio
           run: git clone https://github.com/thestk/rtaudio ; cd rtaudio ; git checkout 2f2fca4502d506abc50f6d4473b2836d24cfb1e3 ; mkdir build ; cd build ; cmake .. ; cmake --build . --config Release ; cmake --install .
@@ -87,7 +87,7 @@ jobs:
           run: brew update
 
         - name: Install dependencies
-          run: brew install fftw glfw airspy airspyhf portaudio hackrf rtl-sdr libbladerf codec2 && pip3 install mako
+          run: brew install fftw glfw airspy airspyhf portaudio hackrf rtl-sdr libbladerf codec2 && pip3 install mako zstd
 
         - name: Install volk
           run: git clone --recursive https://github.com/gnuradio/volk && cd volk && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release .. && make -j3 && sudo make install && cd ../../

core/CMakeLists.txt

@@ -71,6 +71,10 @@ if (MSVC)
     # WinSock2
     target_link_libraries(sdrpp_core PUBLIC wsock32 ws2_32)
 
+    # ZSTD
+    find_package(zstd CONFIG REQUIRED)
+    target_link_libraries(sdrpp_core PUBLIC zstd::libzstd_shared)
+
 else()
     find_package(PkgConfig)
     find_package(OpenGL REQUIRED)
@@ -78,12 +82,14 @@ else()
     pkg_check_modules(FFTW3 REQUIRED fftw3f)
     pkg_check_modules(VOLK REQUIRED volk)
     pkg_check_modules(GLFW3 REQUIRED glfw3)
+    pkg_check_modules(LIBZSTD REQUIRED libzstd)
 
     target_include_directories(sdrpp_core PUBLIC
         ${OPENGL_INCLUDE_DIRS}
         ${FFTW3_INCLUDE_DIRS}
         ${GLFW3_INCLUDE_DIRS}
         ${VOLK_INCLUDE_DIRS}
+        ${LIBZSTD_INCLUDE_DIRS}
     )
     
     target_link_directories(sdrpp_core PUBLIC
@@ -91,6 +97,7 @@ else()
         ${FFTW3_LIBRARY_DIRS}
         ${GLFW3_LIBRARY_DIRS}
         ${VOLK_LIBRARY_DIRS}
+        ${LIBZSTD_LIBRARY_DIRS}
     )
 
     target_link_libraries(sdrpp_core PUBLIC
@@ -98,6 +105,7 @@ else()
         ${FFTW3_LIBRARIES}
         ${GLFW3_LIBRARIES}
         ${VOLK_LIBRARIES}
+        ${LIBZSTD_LIBRARIES}
     )
 
     if (NOT USE_INTERNAL_LIBCORRECT)

core/src/dsp/compression.h

@@ -12,11 +12,16 @@ namespace dsp {
     public:
         DynamicRangeCompressor() {}
 
-        DynamicRangeCompressor(stream<complex_t>* in, PCMType pcmType) { init(in, pcmType); }
+        DynamicRangeCompressor(stream<complex_t>* in, PCMType pcmType) {
+            init(in, pcmType);
+        }
 
         void init(stream<complex_t>* in, PCMType pcmType) {
             _in = in;
             _pcmType = pcmType;
+
+            out.setBufferSize((sizeof(dsp::complex_t) * STREAM_BUFFER_SIZE) + 8);
+
             generic_block<DynamicRangeCompressor>::registerInput(_in);
             generic_block<DynamicRangeCompressor>::registerOutput(&out);
             generic_block<DynamicRangeCompressor>::_block_init = true;

core/src/dsp/stream.h

@@ -31,6 +31,13 @@ namespace dsp {
             volk_free(readBuf);
         }
 
+        void setBufferSize(int samples) {
+            volk_free(writeBuf);
+            volk_free(readBuf);
+            writeBuf = (T*)volk_malloc(samples * sizeof(T), volk_get_alignment());
+            readBuf = (T*)volk_malloc(samples * sizeof(T), volk_get_alignment());
+        }
+
         bool swap(int size) {
             {
                 // Wait to either swap or stop

core/src/server.cpp

@@ -10,6 +10,7 @@
 #include <gui/smgui.h>
 #include <utils/optionlist.h>
 #include <dsp/compression.h>
+#include <zstd.h>
 
 namespace server {
     dsp::stream<dsp::complex_t> dummyInput;
@@ -35,11 +36,14 @@ namespace server {
 
     SmGui::DrawListElem dummyElem;
 
+    ZSTD_CCtx* cctx;
+
     net::Listener listener;
 
     OptionList<std::string, std::string> sourceList;
     int sourceId = 0;
     bool running = false;
+    bool compression = false;
     double sampleRate = 1000000.0;
 
     int main() {
@@ -68,9 +72,8 @@ namespace server {
         bb_pkt_hdr = (PacketHeader*)bbuf;
         bb_pkt_data = &bbuf[sizeof(PacketHeader)];
 
-        // Terminate config manager
+        // Initialize compressor
-        core::configManager.disableAutoSave();
+        cctx = ZSTD_createCCtx();
-        core::configManager.save();
 
         core::configManager.acquire();
         std::string modulesDir = core::configManager.conf["modulesDirectory"];
@@ -183,6 +186,7 @@ namespace server {
         // Perform settings reset
         sigpath::sourceManager.stop();
         comp.setPCMType(dsp::PCM_TYPE_I16);
+        compression = false;
 
         sendSampleRate(sampleRate);
 
@@ -218,14 +222,19 @@ namespace server {
     }
 
     void _testServerHandler(uint8_t* data, int count, void* ctx) {
-        // Build data packet
+        // Compress data if needed and fill out header fields
-        PacketHeader* hdr = (PacketHeader*)bbuf;
+        if (compression) {
-        hdr->type = PACKET_TYPE_BASEBAND;
+            bb_pkt_hdr->type = PACKET_TYPE_BASEBAND_COMPRESSED;
-        hdr->size = sizeof(PacketHeader) + count;
+            bb_pkt_hdr->size = sizeof(PacketHeader) + (uint32_t)ZSTD_compressCCtx(cctx, &bbuf[sizeof(PacketHeader)], SERVER_MAX_PACKET_SIZE, data, count, 1);
+        }
+        else {
+            bb_pkt_hdr->type = PACKET_TYPE_BASEBAND;
+            bb_pkt_hdr->size = sizeof(PacketHeader) + count;
             memcpy(&bbuf[sizeof(PacketHeader)], data, count);
+        }
 
         // Write to network
-        if (client && client->isOpen()) { client->write(hdr->size, bbuf); }
+        if (client && client->isOpen()) { client->write(bb_pkt_hdr->size, bbuf); }
     }
 
     void setInput(dsp::stream<dsp::complex_t>* stream) {
@@ -281,6 +290,9 @@ namespace server {
             dsp::PCMType type = (dsp::PCMType)*(uint8_t*)data;
             comp.setPCMType(type);
         }
+        else if (cmd == COMMAND_SET_COMPRESSION && len == 1) {
+            compression = *(uint8_t*)data;
+        }
         else {
             spdlog::error("Invalid Command: {0} (len = {1})", cmd, len);
             sendError(ERROR_INVALID_COMMAND);

core/src/server_protocol.h

@@ -11,6 +11,7 @@ namespace server {
         PACKET_TYPE_COMMAND,
         PACKET_TYPE_COMMAND_ACK,
         PACKET_TYPE_BASEBAND,
+        PACKET_TYPE_BASEBAND_COMPRESSED,
         PACKET_TYPE_VFO,
         PACKET_TYPE_FFT,
         PACKET_TYPE_ERROR
@@ -25,6 +26,7 @@ namespace server {
         COMMAND_SET_FREQUENCY,
         COMMAND_GET_SAMPLERATE,
         COMMAND_SET_SAMPLE_TYPE,
+        COMMAND_SET_COMPRESSION,
 
         // Server to client
         COMMAND_SET_SAMPLERATE = 0x80,

docker_builds/debian_bullseye/do_build.sh

@@ -4,7 +4,7 @@ cd /root
 
 # Install dependencies and tools
 apt update
-apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
+apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
             libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
             libcodec2-dev
 

docker_builds/debian_buster/do_build.sh

@@ -4,7 +4,7 @@ cd /root
 
 # Install dependencies and tools
 apt update
-apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
+apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
             libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
             libcodec2-dev
 

docker_builds/debian_sid/do_build.sh

@@ -4,7 +4,7 @@ cd /root
 
 # Install dependencies and tools
 apt update
-apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
+apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
             libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
             libcodec2-dev
 

docker_builds/raspios_bullseye/Dockerfile

@@ -1,4 +0,0 @@
-FROM ubuntu:latest
-ENV DEBIAN_FRONTEND=noninteractive
-COPY do_build.sh /root
-RUN chmod +x /root/do_build.sh

docker_builds/raspios_bullseye/do_build.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-set -e
-cd /root
-
-# Install tools
-apt update
-apt install -y wget p7zip-full qemu-user-static gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf build-essential cmake pkg-config
-
-# Download and extract raspberry pi image
-wget https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2021-11-08/2021-10-30-raspios-bullseye-armhf-lite.zip
-7z x 2021-10-30-raspios-bullseye-armhf-lite.zip
-7z x 2021-10-30-raspios-bullseye-armhf-lite.img
-
-# Expand and mount rootfs image
-dd if=/dev/zero bs=1G count=1 >> 1.img
-e2fsck -p -f 1.img
-resize2fs 1.img
-mount 1.img /mnt
-
-# Copy qemu to the rootfs
-cp /usr/bin/qemu-arm-static /mnt/bin/
-
-# Inject script to install dependencies
-echo 'export DEBIAN_FRONTEND=noninteractive' >> /mnt/root/prepare.sh
-echo 'apt update --allow-releaseinfo-change' >> /mnt/root/prepare.sh
-echo 'apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \' >> /mnt/root/prepare.sh
-echo '            libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \' >> /mnt/root/prepare.sh
-echo '            libcodec2-dev' >> /mnt/root/prepare.sh
-
-# Run prepare.sh script
-chroot /mnt /bin/qemu-arm-static /bin/bash /root/prepare.sh
-
-# Setup environment variables
-export PKG_CONFIG_PATH=''
-export PKG_CONFIG_LIBDIR='/mnt/usr/lib/arm-linux-gnueabihf/pkgconfig:/mnt/usr/lib/pkgconfig:/mnt/usr/share/pkgconfig'
-export PKG_CONFIG_SYSROOT_DIR='/mnt'
-
-# Build SDR++
-cd SDRPlusPlus
-mkdir build
-cd build
-cmake .. -DCMAKE_TOOLCHAIN_FILE=../docker_builds/raspios_bullseye/rpi_toolchain.cmake -DOPT_BUILD_BLADERF_SOURCE=ON -DOPT_BUILD_LIMESDR_SOURCE=ON -DOPT_BUILD_NEW_PORTAUDIO_SINK=ON -DOPT_BUILD_M17_DECODER=ON
-make VERBOSE=1 -j2
-
-# Create deb
-cd ..
-sh make_debian_package.sh ./build 'libfftw3-dev, libglfw3-dev, libvolk2-dev, librtaudio-dev'
-mv sdrpp_debian_amd64.deb sdrpp_raspios_arm32.deb

docker_builds/raspios_bullseye/rpi_toolchain.cmake

@@ -1,12 +0,0 @@
-set(CMAKE_SYSTEM_NAME Linux)
-set(CMAKE_SYSTEM_PROCESSOR arm)
-
-set(CMAKE_SYSROOT /mnt)
-
-set(CMAKE_C_COMPILER /usr/bin/arm-linux-gnueabihf-gcc)
-set(CMAKE_CXX_COMPILER /usr/bin/arm-linux-gnueabihf-g++)
-
-set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
-set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
-set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
-set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

docker_builds/ubuntu_bionic/do_build.sh

@@ -10,7 +10,7 @@ echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://ap
 apt update
 
 # Install dependencies and tools
-apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libsoapysdr-dev libairspy-dev \
+apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libzstd-dev libsoapysdr-dev libairspy-dev \
             libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
             libcodec2-dev
 

docker_builds/ubuntu_focal/do_build.sh

@@ -4,7 +4,7 @@ cd /root
 
 # Install dependencies and tools
 apt update
-apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
+apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
             libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
             libcodec2-dev
 

docker_builds/ubuntu_hirsute/do_build.sh

@@ -4,7 +4,7 @@ cd /root
 
 # Install dependencies and tools
 apt update
-apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
+apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
             libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
             libcodec2-dev
 

docker_builds/ubuntu_impish/do_build.sh

@@ -4,7 +4,7 @@ cd /root
 
 # Install dependencies and tools
 apt update
-apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
+apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk2-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
             libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget portaudio19-dev \
             libcodec2-dev
 

readme.md

@@ -78,7 +78,8 @@ brew install \
   portaudio \
   rtl-sdr \
   soapyrtlsdr \
-  volk
+  volk \
+  zstd
 mkdir build
 cd build
 cmake .. \
@@ -115,6 +116,7 @@ After this, install the following dependencies using vcpkg:
 
 * fftw3
 * glfw3
+* zstd
 
 You are probably going to build in 64 bit so make sure vcpkg installs the correct versions using `.\vcpkg.exe install <package>:x64-windows`
 
@@ -221,6 +223,7 @@ you can disable it using the module parameter listed in the table below
 * fftw3
 * glfw
 * libvolk
+* zstd
 
 Next install dependencies based on the modules you wish to build (See previous step)
 

rpi_install.sh

@@ -5,7 +5,7 @@ set -e
 
 echo "Installing dependencies"
 sudo apt update
-sudo apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
+sudo apt install -y build-essential cmake git libfftw3-dev libglfw3-dev libvolk1-dev libzstd-dev libsoapysdr-dev libairspyhf-dev libairspy-dev \
             libiio-dev libad9361-dev librtaudio-dev libhackrf-dev librtlsdr-dev libbladerf-dev liblimesuite-dev p7zip-full wget
 
 echo "Preparing build"

source_modules/sdrpp_server_source/src/main.cpp

@@ -195,10 +195,17 @@ private:
                 config.release(true);
             }
             
-            bool dummy = false;
+            if (ImGui::Checkbox("Compression", &_this->compression)) {
+                _this->client->setCompression(_this->compression);
+
+                // Save config
+                config.acquire();
+                config.conf["servers"][_this->devConfName]["compression"] = _this->compression;
+                config.release(true);
+            }
+
+            bool dummy = true;
             style::beginDisabled();
-            ImGui::Checkbox("Compression", &dummy);
-            dummy = true;
             ImGui::Checkbox("Full IQ", &dummy);
             style::endDisabled();
 
@@ -237,9 +244,13 @@ private:
             std::string key = config.conf["servers"][devConfName]["sampleType"];
             if (sampleTypeList.keyExists(key)) { sampleTypeId = sampleTypeList.keyId(key); }
         }
+        if (config.conf["servers"][devConfName].contains("compression")) {
+            compression = config.conf["servers"][devConfName]["compression"];
+        }
 
         // Set settings
         client->setSampleType(sampleTypeList[sampleTypeId]);
+        client->setCompression(compression);
     }
 
     std::string name;
@@ -261,6 +272,7 @@ private:
 
     OptionList<std::string, dsp::PCMType> sampleTypeList;
     int sampleTypeId;
+    bool compression = false;
 
     server::Client client;
 };

source_modules/sdrpp_server_source/src/sdrpp_server_client.cpp

@@ -26,7 +26,11 @@ namespace server {
         s_cmd_hdr = (CommandHeader*)s_pkt_data;
         s_cmd_data = &sbuffer[sizeof(PacketHeader) + sizeof(CommandHeader)];
 
+        // Initialize decompressor
+        dctx = ZSTD_createDCtx();
+
         // Initialize DSP
+        decompIn.setBufferSize((sizeof(dsp::complex_t) * STREAM_BUFFER_SIZE) + 8);
         decomp.init(&decompIn);
         link.init(&decomp.out, output);
         decomp.start();
@@ -43,6 +47,7 @@ namespace server {
 
     ClientClass::~ClientClass() {
         close();
+        ZSTD_freeDCtx(dctx);
         delete[] rbuffer;
         delete[] sbuffer;
     }
@@ -108,6 +113,11 @@ namespace server {
         sendCommand(COMMAND_SET_SAMPLE_TYPE, 1);
     }
 
+    void ClientClass::setCompression(bool enabled) {
+         s_cmd_data[0] = enabled;
+        sendCommand(COMMAND_SET_COMPRESSION, 1);
+    }
+
     void ClientClass::start() {
         if (!client || !client->isOpen()) { return; }
         sendCommand(COMMAND_START, 0);
@@ -189,6 +199,10 @@ namespace server {
             memcpy(_this->decompIn.writeBuf, &buf[sizeof(PacketHeader)], _this->r_pkt_hdr->size - sizeof(PacketHeader));
             _this->decompIn.swap(_this->r_pkt_hdr->size - sizeof(PacketHeader));
         }
+        else if (_this->r_pkt_hdr->type == PACKET_TYPE_BASEBAND_COMPRESSED) {
+            size_t outCount = ZSTD_decompressDCtx(_this->dctx, _this->decompIn.writeBuf, STREAM_BUFFER_SIZE, _this->r_pkt_data, _this->r_pkt_hdr->size - sizeof(PacketHeader));
+            if (outCount) { _this->decompIn.swap(outCount); };
+        }
         else if (_this->r_pkt_hdr->type == PACKET_TYPE_ERROR) {
             spdlog::error("SDR++ Server Error: {0}", buf[sizeof(PacketHeader)]);
         }

source_modules/sdrpp_server_source/src/sdrpp_server_client.h

@@ -11,6 +11,7 @@
 #include <dsp/compression.h>
 #include <dsp/sink.h>
 #include <dsp/link.h>
+#include <zstd.h>
 
 #define RFSPACE_MAX_SIZE                8192
 #define RFSPACE_HEARTBEAT_INTERVAL_MS   1000
@@ -85,6 +86,7 @@ namespace server {
         double getSampleRate();
         
         void setSampleType(dsp::PCMType type);
+        void setCompression(bool enabled);
 
         void start();
         void stop();
@@ -133,6 +135,8 @@ namespace server {
         SmGui::DrawList dl;
         std::mutex dlMtx;
 
+        ZSTD_DCtx* dctx;
+
         double currentSampleRate = 1000000.0;
     };