Looking for Linux Fixes

.devcontainer/Dockerfile

@@ -0,0 +1,22 @@
+FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04
+# Use the official devcontainer base image for Ubuntu 24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Free Pascal, fpc-source and Lazarus non-interactively and clean up
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+    fpc \
+    fpc-source \
+    lazarus \
+    php \
+    build-essential \
+ || true \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Ensure workspace folder exists (devcontainers typically mount the repo)
+WORKDIR /workspaces/neural-api
+
+# Use the default non-root user provided by the base image
+USER vscode

.devcontainer/devcontainer.json

@@ -0,0 +1,19 @@
+{
+  "name": "neural-api: Free Pascal",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  "workspaceFolder": "/workspaces/neural-api",
+  "postCreateCommand": "fpc -v || true",
+  "remoteUser": "vscode",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "alefragnani.pascal",
+        "davidanson.vscode-markdownlint",
+        "ms-vscode.vscode-container-tools"
+      ]
+    }
+  }
+}

.github/workflows/tests.yml

@@ -0,0 +1,55 @@
+name: Tests
+
+on:
+  push:
+    branches: [ master, main ]
+  pull_request:
+    branches: [ master, main ]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Install Free Pascal Compiler and LCL
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y fpc lcl
+
+    - name: Find LazUtils path
+      id: lazutils
+      run: |
+        # Find the correct LazUtils path for the installed version
+        # Search in common Lazarus installation locations
+        LAZUTILS_PATH=$(find /usr/lib/lazarus /usr/share/lazarus -name "utf8process.ppu" -printf "%h\n" 2>/dev/null | head -1)
+        if [ -z "$LAZUTILS_PATH" ]; then
+          echo "Warning: LazUtils not found, falling back to basic tests only"
+          echo "path=" >> $GITHUB_OUTPUT
+          echo "found=false" >> $GITHUB_OUTPUT
+        else
+          echo "Found LazUtils at: ${LAZUTILS_PATH}"
+          echo "path=${LAZUTILS_PATH}" >> $GITHUB_OUTPUT
+          echo "found=true" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Build Tests
+      run: |
+        mkdir -p bin
+        cd tests
+        if [ -n "${{ steps.lazutils.outputs.path }}" ]; then
+          fpc -Mobjfpc -Fu../neural -Fu${{ steps.lazutils.outputs.path }} -FE../bin -o../bin/RunTests RunTests.lpr
+        else
+          echo "Building without LazUtils - threading tests will be skipped"
+          fpc -Mobjfpc -Fu../neural -FE../bin -o../bin/RunTests RunTests.lpr
+        fi
+
+    - name: Run Tests
+      run: |
+        cd bin
+        ./RunTests --all --format=plain

.gitignore

@@ -2,3 +2,13 @@ experiments
 opencl
 libs
 readme.txt
+
+# Build artifacts
+bin/
+*.o
+*.ppu
+*.exe
+*.compiled
+*.bak
+backup/
+lib/

docs/videos.md

@@ -0,0 +1,8 @@
+### Coding Neural Networks Easy - Youtube Videos
+[![Watch the video](https://img.youtube.com/vi/aIy1S7clhQo/0.jpg)](https://youtu.be/aIy1S7clhQo) | [![Watch the video](https://img.youtube.com/vi/q56NcgUiAAk/0.jpg)](https://youtu.be/q56NcgUiAAk) | [![Watch the video](https://img.youtube.com/vi/PdNTgI_qSyo/0.jpg)](https://youtu.be/PdNTgI_qSyo)
+--------------------------- | ------------------------------------- | -------------------------
+Basics of Neural Networks in Pascal - Loading and Saving | Neural Networks for Absolute Beginners! Learning a Simple Function | Coding a Neural Network in Pascal that Learns to Calculate the Hypotenuse
+[![Watch the video](https://img.youtube.com/vi/tODsv6Ks2DM/0.jpg)](https://youtu.be/tODsv6Ks2DM) | [![Watch the video](https://img.youtube.com/vi/f4T9IB-He_k/0.jpg)](https://youtu.be/f4T9IB-He_k) | [![Watch the video](https://img.youtube.com/vi/o-8NuoSsdck/0.jpg)](https://youtu.be/o-8NuoSsdck)
+Pre-trained Neural Networks & Transfer Learning with Pascal's CAI Neural API | Coding a Neural Network in Pascal that Learns the OR Boolean Operation | A Dive into Identity Shortcut Connection - The ResNet building block
+[![Watch the video](https://img.youtube.com/vi/SEvWB7k8uy0/0.jpg)](https://youtu.be/SEvWB7k8uy0) | [![Watch the video](https://img.youtube.com/vi/3QwIaAsDmJw/0.jpg)](https://youtu.be/3QwIaAsDmJw) | [![Watch the video](https://img.youtube.com/vi/VH6v3D5cxxs/0.jpg)](https://youtu.be/VH6v3D5cxxs)
+Increasing Image Resolution with Neural Networks | Ultra Fast Single Precision Floating Point Computing | AVX and AVX2 Code Optimization

docs/videos/README.md

@@ -0,0 +1,7 @@
+[![Watch the video](https://img.youtube.com/vi/aIy1S7clhQo/0.jpg)](https://youtu.be/aIy1S7clhQo) | [![Watch the video](https://img.youtube.com/vi/q56NcgUiAAk/0.jpg)](https://youtu.be/q56NcgUiAAk) | [![Watch the video](https://img.youtube.com/vi/PdNTgI_qSyo/0.jpg)](https://youtu.be/PdNTgI_qSyo)
+--------------------------- | ------------------------------------- | -------------------------
+Basics of Neural Networks in Pascal - Loading and Saving | Neural Networks for Absolute Beginners! Learning a Simple Function | Coding a Neural Network in Pascal that Learns to Calculate the Hypotenuse
+[![Watch the video](https://img.youtube.com/vi/tODsv6Ks2DM/0.jpg)](https://youtu.be/tODsv6Ks2DM) | [![Watch the video](https://img.youtube.com/vi/f4T9IB-He_k/0.jpg)](https://youtu.be/f4T9IB-He_k) | [![Watch the video](https://img.youtube.com/vi/o-8NuoSsdck/0.jpg)](https://youtu.be/o-8NuoSsdck)
+Pre-trained Neural Networks & Transfer Learning with Pascal's CAI Neural API | Coding a Neural Network in Pascal that Learns the OR Boolean Operation | A Dive into Identity Shortcut Connection - The ResNet building block
+[![Watch the video](https://img.youtube.com/vi/SEvWB7k8uy0/0.jpg)](https://youtu.be/SEvWB7k8uy0) | [![Watch the video](https://img.youtube.com/vi/3QwIaAsDmJw/0.jpg)](https://youtu.be/3QwIaAsDmJw) | [![Watch the video](https://img.youtube.com/vi/VH6v3D5cxxs/0.jpg)](https://youtu.be/VH6v3D5cxxs)
+Increasing Image Resolution with Neural Networks | Ultra Fast Single Precision Floating Point Computing | AVX and AVX2 Code Optimization

examples/CaiOptimizedDenseNet/CaiOptimizedDenseNet.lpr

@@ -74,7 +74,7 @@   TTestCNNAlgo = class(TCustomApplication)
     end;
 
     bSeparable := HasOption('s', 'separable');
-    bPaddingCropping := HasOption('p', 'padding');
+    bPaddingCropping := true; // HasOption('p', 'padding');
 
     iInnerConvNum := 12;
     if HasOption('c', 'convolutions') then
@@ -123,16 +123,37 @@   TTestCNNAlgo = class(TCustomApplication)
     NN.AddDenseNetBlockCAI(iInnerConvNum div 6, iConvNeuronCount, {supressBias=}1, TNNetConvolutionReLU, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeNorm=}nil, {pAfterNorm=}nil, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0, {RandomAdd=}1, {RandomMul=}1);
     NN.AddCompression(1);
     NN.AddLayer( TNNetMaxPool.Create(2) );
-    NN.AddDenseNetBlockCAI(iInnerConvNum div 3, iConvNeuronCount, {supressBias=}1, TNNetConvolutionReLU, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeNorm=}nil, {pAfterNorm=}nil, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0, {RandomAdd=}1, {RandomMul=}1);
+    //NN.AddDenseNetBlockCAI(iInnerConvNum div 3, iConvNeuronCount, {supressBias=}1, TNNetConvolutionReLU, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeNorm=}nil, {pAfterNorm=}nil, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0, {RandomAdd=}1, {RandomMul=}1);
+    NN.AddDenseNetBlockCAI(iInnerConvNum div 3, iConvNeuronCount, {supressBias=}1, TNNetConvolutionLinear, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeBottleNeck=}nil, {pAfterBottleNeck=}TNNetSwish6, {pBeforeConv=}nil, {pAfterConv=}TNNetSwish6, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0);
     NN.AddCompression(1);
     NN.AddLayer( TNNetMaxPool.Create(2) );
-    NN.AddDenseNetBlockCAI(iInnerConvNum div 3, iConvNeuronCount, {IsSeparable=}1, TNNetConvolutionReLU, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeNorm=}nil, {pAfterNorm=}nil, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0, {RandomAdd=}1, {RandomMul=}1);
+    //NN.AddDenseNetBlockCAI(iInnerConvNum div 3, iConvNeuronCount, {IsSeparable=}1, TNNetConvolutionReLU, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeNorm=}nil, {pAfterNorm=}nil, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0, {RandomAdd=}1, {RandomMul=}1);
+    NN.AddDenseNetBlockCAI(iInnerConvNum div 3, iConvNeuronCount, {supressBias=}1, TNNetConvolutionLinear, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeBottleNeck=}nil, {pAfterBottleNeck=}TNNetSwish6, {pBeforeConv=}nil, {pAfterConv=}TNNetSwish6, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0);
     NN.AddCompression(1);
+    NN.AddLayer( TNNetDropout.Create(0.5) );
+    NN.AddLayer( TNNetMaxChannel.Create() );
+    NN.AddLayer( TNNetFullConnectLinear.Create(NumClasses) );
+    NN.AddLayer( TNNetSoftMax.Create({SkipBackpropDerivative=}1) );
+    NN.Layers[ NN.GetFirstImageNeuronalLayerIdx() ].InitBasicPatterns();
+    (*
+    // First block shouldn't be separable.
+    NN.AddDenseNetBlockCAI(iInnerConvNum div 6, iConvNeuronCount, {supressBias=}0, TNNetConvolutionReLU, {IsSeparable=}false, {HasMovingNorm=}HasMovingNorm, {pBeforeBottleNeck=}nil, {pAfterBottleNeck=}nil, {pBeforeConv=}nil, {pAfterConv=}TNNetReLU, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0);
+    NN.AddDenseNetBlockCAI(iInnerConvNum div 6, iConvNeuronCount, {supressBias=}0, TNNetConvolutionReLU, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeBottleNeck=}nil, {pAfterBottleNeck=}nil, {pBeforeConv=}nil, {pAfterConv=}TNNetReLU, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0);
+    NN.AddCompression(1);
+    NN.AddLayer( TNNetMaxPool.Create(2) );
+    NN.AddDenseNetBlockCAI(iInnerConvNum div 3, iConvNeuronCount, {supressBias=}0, TNNetConvolutionLinear, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeBottleNeck=}nil, {pAfterBottleNeck=}TNNetSwish, {pBeforeConv=}nil, {pAfterConv=}TNNetSwish, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0);
+    NN.AddCompression(1);
+    NN.AddLayer( TNNetSwish.Create() );
+    NN.AddLayer( TNNetMaxPool.Create(2) );
+    NN.AddDenseNetBlockCAI(iInnerConvNum div 3, iConvNeuronCount, {IsSeparable=}0, TNNetConvolutionLinear, {IsSeparable=}bSeparable, {HasMovingNorm=}HasMovingNorm, {pBeforeBottleNeck=}nil, {pAfterBottleNeck=}TNNetSwish, {pBeforeConv=}nil, {pAfterConv=}TNNetSwish, {BottleNeck=}iBottleneck, {Compression=}0, {Dropout=}0);
+    NN.AddCompression(1);
+    NN.AddLayer( TNNetSwish.Create() );
     NN.AddLayer( TNNetDropout.Create(0.25) );
     NN.AddLayer( TNNetMaxChannel.Create() );
     NN.AddLayer( TNNetFullConnectLinear.Create(NumClasses) );
     NN.AddLayer( TNNetSoftMax.Create() );
     NN.Layers[ NN.GetFirstImageNeuronalLayerIdx() ].InitBasicPatterns();
+    *)
 
     WriteLn('Learning rate set to: [',fLearningRate:7:5,']');
     WriteLn('Inertia set to: [',fInertia:7:5,']');
@@ -211,7 +232,7 @@   TTestCNNAlgo = class(TCustomApplication)
       ' -c : defines the number of convolutions. Default is 12.', sLineBreak,
       ' -b : defines the bottleneck. Default is 32.', sLineBreak,
       ' -n : defines convolutional neurons (growth rate). Default is 32.', sLineBreak,
-      ' -p : enables padding and cropping data augmentation.', sLineBreak,
+//      ' -p : enables padding and cropping data augmentation.', sLineBreak,
       ' https://github.com/joaopauloschuler/neural-api/tree/master/examples/CaiOptimizedDenseNet',sLineBreak,
       ' More info at:',sLineBreak,
       '   https://github.com/joaopauloschuler/neural-api'

examples/CaiOptimizedDenseNet/CaiOptimizedDenseNet48.lpr

@@ -124,7 +124,7 @@   TTestCNNAlgo = class(TCustomApplication)
     NN.AddLayer( TNNetDropout.Create(0.25) );
     NN.AddLayer( TNNetMaxChannel.Create() );
     NN.AddLayer( TNNetFullConnectLinear.Create(NumClasses) );
-    NN.AddLayer( TNNetSoftMax.Create() );
+    NN.AddLayer( TNNetSoftMax.Create({SkipBackpropDerivative=}1) );
     NN.Layers[ NN.GetFirstImageNeuronalLayerIdx() ].InitBasicPatterns();
 
     WriteLn('Learning rate set to: [',fLearningRate:7:5,']');

examples/CaiOptimizedDenseNet/kOptimizedDenseNet.lpi

@@ -0,0 +1,174 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CONFIG>
+  <ProjectOptions>
+    <Version Value="11"/>
+    <PathDelim Value="\"/>
+    <General>
+      <Flags>
+        <MainUnitHasCreateFormStatements Value="False"/>
+      </Flags>
+      <SessionStorage Value="InProjectDir"/>
+      <MainUnit Value="0"/>
+      <Title Value="CIFAR-10 Classification Example"/>
+      <UseAppBundle Value="False"/>
+      <ResourceType Value="res"/>
+    </General>
+    <i18n>
+      <EnableI18N LFM="False"/>
+    </i18n>
+    <BuildModes Count="3">
+      <Item1 Name="Default" Default="True"/>
+      <Item2 Name="Debug">
+        <CompilerOptions>
+          <Version Value="11"/>
+          <PathDelim Value="\"/>
+          <Target>
+            <Filename Value="..\..\bin\$(TargetCPU)-$(TargetOS)\bin\kOptimizedDenseNet"/>
+          </Target>
+          <SearchPaths>
+            <IncludeFiles Value="..\..\neural;$(ProjOutDir)"/>
+            <OtherUnitFiles Value="..\..\neural;$(LazarusDir)\lcl\units\$(TargetCPU)-$(TargetOS);$(LazarusDir)\components\lazutils\lib\$(TargetCPU)-$(TargetOS)"/>
+            <UnitOutputDirectory Value="..\..\bin\$(TargetCPU)-$(TargetOS)\units"/>
+          </SearchPaths>
+          <Parsing>
+            <SyntaxOptions>
+              <IncludeAssertionCode Value="True"/>
+            </SyntaxOptions>
+          </Parsing>
+          <CodeGeneration>
+            <Checks>
+              <IOChecks Value="True"/>
+              <RangeChecks Value="True"/>
+              <OverflowChecks Value="True"/>
+              <StackChecks Value="True"/>
+            </Checks>
+            <VerifyObjMethodCallValidity Value="True"/>
+          </CodeGeneration>
+          <Linking>
+            <Debugging>
+              <DebugInfoType Value="dsDwarf2Set"/>
+              <UseValgrind Value="True"/>
+              <UseExternalDbgSyms Value="True"/>
+            </Debugging>
+          </Linking>
+          <Other>
+            <CustomOptions Value="-dDebug
+-dAVX"/>
+            <OtherDefines Count="3">
+              <Define0 Value="Debug"/>
+              <Define1 Value="AVX"/>
+              <Define2 Value="MakeQuick"/>
+            </OtherDefines>
+          </Other>
+        </CompilerOptions>
+      </Item2>
+      <Item3 Name="Release">
+        <CompilerOptions>
+          <Version Value="11"/>
+          <PathDelim Value="\"/>
+          <Target>
+            <Filename Value="..\..\bin\$(TargetCPU)-$(TargetOS)\bin\kOptimizedDenseNet"/>
+          </Target>
+          <SearchPaths>
+            <IncludeFiles Value="..\..\neural;$(ProjOutDir)"/>
+            <OtherUnitFiles Value="..\..\neural;$(LazarusDir)\lcl\units\$(TargetCPU)-$(TargetOS);$(LazarusDir)\components\lazutils\lib\$(TargetCPU)-$(TargetOS)"/>
+            <UnitOutputDirectory Value="..\..\bin\$(TargetCPU)-$(TargetOS)\units"/>
+          </SearchPaths>
+          <CodeGeneration>
+            <SmartLinkUnit Value="True"/>
+            <Optimizations>
+              <OptimizationLevel Value="3"/>
+            </Optimizations>
+          </CodeGeneration>
+          <Linking>
+            <Debugging>
+              <GenerateDebugInfo Value="False"/>
+            </Debugging>
+            <LinkSmart Value="True"/>
+          </Linking>
+          <Other>
+            <CustomOptions Value="-dRelease
+-dAVX"/>
+            <OtherDefines Count="5">
+              <Define0 Value="Release"/>
+              <Define1 Value="Debug"/>
+              <Define2 Value="CheckRange"/>
+              <Define3 Value="AVX2"/>
+              <Define4 Value="AVX"/>
+            </OtherDefines>
+          </Other>
+        </CompilerOptions>
+      </Item3>
+    </BuildModes>
+    <PublishOptions>
+      <Version Value="2"/>
+    </PublishOptions>
+    <RunParams>
+      <local>
+        <CommandLineParams Value="-a 3"/>
+      </local>
+      <FormatVersion Value="2"/>
+      <Modes Count="1">
+        <Mode0 Name="default">
+          <local>
+            <CommandLineParams Value="-a 3"/>
+          </local>
+        </Mode0>
+      </Modes>
+    </RunParams>
+    <RequiredPackages Count="1">
+      <Item1>
+        <PackageName Value="multithreadprocslaz"/>
+      </Item1>
+    </RequiredPackages>
+    <Units Count="1">
+      <Unit0>
+        <Filename Value="kOptimizedDenseNet.lpr"/>
+        <IsPartOfProject Value="True"/>
+      </Unit0>
+    </Units>
+  </ProjectOptions>
+  <CompilerOptions>
+    <Version Value="11"/>
+    <PathDelim Value="\"/>
+    <Target>
+      <Filename Value="..\..\bin\$(TargetCPU)-$(TargetOS)\bin\kOptimizedDenseNet"/>
+    </Target>
+    <SearchPaths>
+      <IncludeFiles Value="..\..\neural;$(ProjOutDir)"/>
+      <OtherUnitFiles Value="..\..\neural;$(LazarusDir)\lcl\units\$(TargetCPU)-$(TargetOS);$(LazarusDir)\components\lazutils\lib\$(TargetCPU)-$(TargetOS)"/>
+      <UnitOutputDirectory Value="..\..\bin\$(TargetCPU)-$(TargetOS)\units"/>
+    </SearchPaths>
+    <CodeGeneration>
+      <Optimizations>
+        <OptimizationLevel Value="3"/>
+      </Optimizations>
+    </CodeGeneration>
+    <Linking>
+      <Debugging>
+        <GenerateDebugInfo Value="False"/>
+      </Debugging>
+    </Linking>
+    <Other>
+      <CustomOptions Value="-dAVX
+-dRelease"/>
+      <OtherDefines Count="2">
+        <Define0 Value="AVX"/>
+        <Define1 Value="Release"/>
+      </OtherDefines>
+    </Other>
+  </CompilerOptions>
+  <Debugging>
+    <Exceptions Count="3">
+      <Item1>
+        <Name Value="EAbort"/>
+      </Item1>
+      <Item2>
+        <Name Value="ECodetoolError"/>
+      </Item2>
+      <Item3>
+        <Name Value="EFOpenError"/>
+      </Item3>
+    </Exceptions>
+  </Debugging>
+</CONFIG>