enable builkite

Author: [email protected]

.buildkite/pipeline.yml

@@ -0,0 +1,94 @@
+# .buildkite/pipeline.yml
+steps:
+  # --------------------------------------------------------
+  # STEP 1 — Get changed files
+  # --------------------------------------------------------
+  - label: ":git: Detect changed files"
+    key: detect-changes
+    commands:
+      - echo "--- Detecting changed files"
+      - git fetch origin main
+      - git diff --name-only origin/main...HEAD > changed_files.txt
+      - echo "Changed files:"
+      - cat changed_files.txt
+      # Save for later steps
+      - buildkite-agent artifact upload changed_files.txt
+
+  # --------------------------------------------------------
+  # STEP 2 — Determine languages based on changed files
+  # --------------------------------------------------------
+  - label: ":mag: Identify languages"
+    key: list-languages
+    depends_on: detect-changes
+    commands:
+      - buildkite-agent artifact download changed_files.txt .
+      - echo "--- Extracting top-level folders"
+      - awk -F/ '{print $1}' changed_files.txt | sort -u > languages.txt
+      - echo "Languages found:"
+      - cat languages.txt
+      - buildkite-agent artifact upload languages.txt
+
+  # --------------------------------------------------------
+  # STEP 3 — Discover frameworks per language
+  # --------------------------------------------------------
+  - label: ":gear: Discover frameworks"
+    key: list-frameworks
+    depends_on: list-languages
+    commands:
+      - buildkite-agent artifact download languages.txt .
+      - echo "--- Listing frameworks per language"
+      - mkdir frameworks
+      - |
+        while read lang; do
+          echo "Finding frameworks in $lang/"
+          find "$lang" -maxdepth 1 -type d ! -path "$lang" -printf "%f\n" \
+            | grep -v '^\.$' > "frameworks/${lang}.txt" || true
+          echo "$lang frameworks:"
+          cat "frameworks/${lang}.txt" || echo "(none)"
+        done < languages.txt
+
+      # Combine all into JSON pairs for clarity
+      - |
+        echo "[" > pairs.json
+        first=true
+        while read lang; do
+          while read fw; do
+            if [ -n "$fw" ]; then
+              if [ "$first" = false ]; then echo "," >> pairs.json; fi
+              echo "{\"language\":\"$lang\",\"framework\":\"$fw\"}" >> pairs.json
+              first=false
+            fi
+          done < "frameworks/${lang}.txt"
+        done < languages.txt
+        echo "]" >> pairs.json
+
+      - echo "Discovered pairs:"
+      - cat pairs.json
+      - buildkite-agent artifact upload pairs.json
+
+  # --------------------------------------------------------
+  # STEP 4 — Dynamically create one job per (language, framework)
+  # --------------------------------------------------------
+  - label: ":rocket: Expand pipeline dynamically"
+    key: expand
+    depends_on: list-frameworks
+    commands:
+      - buildkite-agent artifact download pairs.json .
+      - echo "--- Expanding pipeline dynamically"
+
+      # Split into individual steps and upload
+      - |
+        jq -c '.[]' pairs.json | while read pair; do
+          lang=$(echo "$pair" | jq -r '.language')
+          fw=$(echo "$pair" | jq -r '.framework')
+
+          cat <<YAML > step.yml
+steps:
+  - label: ":test_tube: Run tests - ${lang}/${fw}"
+    command:
+      - echo "Running tests for ${lang}/${fw}"
+      - ./scripts/run-tests.sh ${lang} ${fw}
+YAML
+          buildkite-agent pipeline upload step.yml
+        done
+

.env

@@ -1,4 +1,5 @@
 DATABASE_URL=postgresql://postgres@localhost/benchmark
 DURATION=15
-CONCURRENCIES=64,256,512
+#CONCURRENCIES=64,256,512
+CONCURRENCIES=64,128,256
 ROUTES=GET:/,GET:/user/0,POST:/user

.tasks/ci.rake

@@ -34,4 +34,4 @@ namespace :ci do
     matrix[:include].uniq!
     puts matrix.to_json
   end
-end
+end

.tasks/collect.rake

@@ -24,52 +24,51 @@ end
 task :collect do
   database = ENV.fetch('DATABASE_URL')
   db = PG.connect(database)
- Dir.glob('*/*/.results/*/**.json').each do |file|
-   pp file
-  info = file.split('/')
-   language = info[0]
-   framework = info[1]
-   concurrency = info[3]
-  res = db.query(
-    'INSERT INTO languages (label) VALUES ($1) ON CONFLICT (label) DO UPDATE SET label = $1 RETURNING id', [language]
-  )
-  language_id = res.first['id']
+  Dir.glob('*/*/.results/*/**.json').each do |file|
+    info = file.split('/')
+    language = info[0]
+    framework = info[1]
+    concurrency = info[3]
+    res = db.query(
+      'INSERT INTO languages (label) VALUES ($1) ON CONFLICT (label) DO UPDATE SET label = $1 RETURNING id', [language]
+    )
+    language_id = res.first['id']
 
-  res = db.query(
-    'INSERT INTO frameworks (language_id, label) VALUES ($1, $2) ON CONFLICT (language_id, label) DO UPDATE SET label = $2 RETURNING id', [
-      language_id, framework
-    ]
-  )
-  framework_id = res.first['id']
-  data = YAML.load_file(file, symbolize_names: true)
+    res = db.query(
+      'INSERT INTO frameworks (language_id, label) VALUES ($1, $2) ON CONFLICT (language_id, label) DO UPDATE SET label = $2 RETURNING id', [
+        language_id, framework
+      ]
+    )
+    framework_id = res.first['id']
+    data = YAML.load_file(file, symbolize_names: true)
 
-      res = db.query(
-        'INSERT INTO concurrencies (level) VALUES ($1) ON CONFLICT (level) DO UPDATE SET level = $1 RETURNING id', [concurrency]
-      )
+    res = db.query(
+      'INSERT INTO concurrencies (level) VALUES ($1) ON CONFLICT (level) DO UPDATE SET level = $1 RETURNING id', [concurrency]
+    )
 
-      concurrency_level_id = res.first['id']
-results = {
-  duration_ms: data.dig(:summary, :total) * 1000,
-  total_requests: -1,
-  total_requests_per_s: data.dig(:summary, :requestsPerSec),
-  total_bytes_received: data.dig(:summary, :totalData),
-  socket_connection_errors: -1,
-  socket_read_errors: -1,
-  socket_write_errors: -1,
-  http_errors: -1,
-  request_timeouts: -1,
-  minimum_latency: -1,
-  average_latency: -1,
-  standard_deviation: -1,
-  percentile50: data.dig(:latencyPercentiles, :p50),
-  percentile75: data.dig(:latencyPercentiles, :p75),
-  percentile90: data.dig(:latencyPercentiles, :p90),
-  percentile99:data.dig(:latencyPercentiles, :p95),
-  'percentile99999': -1,
-}
-results.each do |key, value|
-  insert(db, framework_id, key, value, concurrency_level_id)
+    concurrency_level_id = res.first['id']
+    results = {
+      duration_ms: data.dig(:summary, :total) * 1000,
+      total_requests: -1,
+      total_requests_per_s: data.dig(:summary, :requestsPerSec),
+      total_bytes_received: data.dig(:summary, :totalData),
+      socket_connection_errors: -1,
+      socket_read_errors: -1,
+      socket_write_errors: -1,
+      http_errors: -1,
+      request_timeouts: -1,
+      minimum_latency: -1,
+      average_latency: -1,
+      standard_deviation: -1,
+      percentile50: data.dig(:latencyPercentiles, :p50),
+      percentile75: data.dig(:latencyPercentiles, :p75),
+      percentile90: data.dig(:latencyPercentiles, :p90),
+      percentile99: data.dig(:latencyPercentiles, :p95),
+      percentile99999: -1
+    }
+    results.each do |key, value|
+      insert(db, framework_id, key, value, concurrency_level_id)
     end
-end
+  end
   db.close
 end

.tasks/db.rake

@@ -29,16 +29,20 @@ end
 namespace :db do
   task :md_export do
     results = JSON.parse(File.read('data.json'))
-    results['metrics'].map { |m| m['framework'] = results['frameworks'].detect {|f| f['id'] == m['framework_id'] }['label']  }
+    results['metrics'].map { |m| m['framework'] = results['frameworks'].detect { |f| f['id'] == m['framework_id'] }['label'] }
+    reqs = []
     results['metrics'].each do |metric|
-       pp "| #{metric['framework']} | #{metric['level']} | #{metric['value']} |" if metric['label'] == 'total_requests_per_s' 
+      if metric['label'] == 'total_requests_per_s'
+        reqs << { framework: metric['framework'], level: metric['level'], value: metric['value'] }
+      end
     end
+    pp reqs.select { |i| i[:level] == 256 }.sort_by { |i| i[:value] }
   end
   task :check_failures do
     results = JSON.parse(File.read('data.json'))
     results['frameworks'].map { _1['label'] }
     failing_frameworks = results['metrics'].filter_map do |row|
-      row['framework_id'] if row['label'] == 'total_requests_per_s' && (row['value']).zero?
+      row['framework_id'] if row['label'] == 'total_requests_per_s' && row['value'].zero?
     end
     list_of = Dir.glob('*/*/config.yaml').map { _1.split('/')[1] }
     $stdout.puts "Failing : #{results['frameworks'].filter_map do |row|

clojure/pedestal/project.clj

@@ -4,7 +4,7 @@
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[org.clojure/clojure "1.12.3"]
-                 [io.pedestal/pedestal.service "0.8.0"]
+                 [io.pedestal/pedestal.service "0.8.1"]
                  [io.pedestal/pedestal.jetty "0.8.1"]]
   :min-lein-version "2.0.0"
   :resource-paths []

config.yaml

@@ -32,7 +32,7 @@ service: |
 providers:
   docker:
     build:
-      - cd {{{language}}}/{{{framework}}} && docker build -f {{{manifest}}} -t {{{language}}}.{{{framework}}}.{{{variant}}} . --no-cache && cd -
+      - cd {{{language}}}/{{{framework}}} && docker build -f {{{manifest}}} -t {{{language}}}.{{{framework}}}.{{{variant}}} . && cd -
 
     metadata:
       - docker run -td {{{language}}}.{{{framework}}}.{{{variant}}} --network host > {{{language}}}/{{{framework}}}/cid-{{{variant}}}.txt