feat: improve ontology download behavior with smart version tracking

- Replace misleading "Downloading..." messages with "Checking..."
- Add clear status indicators: ✓ Up-to-date, ⟳ Updated, ✅ Downloaded
- Add HTTP HEAD checks to detect remote changes without downloading
- Store remote metadata (ETag, Content-Length) for efficient change detection
- Track last_checked timestamp even when files are up-to-date
- Improve version tracking with check_only mode
- Add better error messages and status reporting

This makes the workflow more efficient by:
- Only downloading when files actually change
- Providing clear feedback about what's happening
- Using HTTP headers to detect changes without full downloads
- Maintaining comprehensive version history

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: jplfaria

scripts/analyze_core_ontologies.py

@@ -194,7 +194,7 @@ def analyze_core_ontologies(repo_path):
             filename = os.path.basename(entry)
             output_path = os.path.join(ontology_data_path, filename)
 
-            print(f"Downloading core ontology: {filename}")
+            print(f"Checking core ontology: {filename}")
             if not download_ontology(entry, output_path, repo_path):
                 print(f"⚠️  Failed to download {filename}, skipping analysis")
                 continue
@@ -272,7 +272,7 @@ def analyze_core_ontologies(repo_path):
             filename = os.path.basename(entry)
             output_path = os.path.join(non_base_dir, filename)
 
-            print(f"Downloading non-base ontology: {filename}")
+            print(f"Checking non-base ontology: {filename}")
             if not download_ontology(entry, output_path, repo_path):
                 print(f"⚠️  Failed to download {filename}, skipping analysis")
                 continue

scripts/enhanced_download.py

@@ -78,6 +78,38 @@ def is_test_mode():
     return 'test' in source_file.lower()
 
 
+def check_remote_changes(url, version_info):
+    """Check if remote file has changed using HTTP HEAD request."""
+    try:
+        response = requests.head(url, timeout=10, allow_redirects=True)
+        response.raise_for_status()
+        
+        # Get remote metadata
+        remote_size = response.headers.get('Content-Length')
+        remote_etag = response.headers.get('ETag', '').strip('"')
+        remote_modified = response.headers.get('Last-Modified')
+        
+        # Compare with stored metadata
+        if version_info:
+            stored_etag = version_info.get('remote_etag')
+            stored_size = version_info.get('remote_size')
+            
+            # If we have ETag, use it for comparison
+            if remote_etag and stored_etag:
+                return remote_etag != stored_etag
+            
+            # Otherwise, check size if available
+            if remote_size and stored_size:
+                return str(remote_size) != str(stored_size)
+        
+        # If no metadata to compare, assume it might have changed
+        return True
+        
+    except requests.exceptions.RequestException:
+        # If HEAD fails, assume we need to check by downloading
+        return True
+
+
 def download_with_retry(url, max_retries=3, timeout=30):
     """Download with exponential backoff retry logic."""
     for attempt in range(max_retries):
@@ -146,8 +178,12 @@ def download_ontology_with_versioning(url, output_path, repo_path, force_downloa
         if not force_download:
             needs_download, reason = should_download(output_path, url, version_file)
             if not needs_download:
-                log_download_attempt(version_dir, filename, "skipped", None, url)
-                return True, "skipped", f"File up to date: {filename}"
+                # Get current checksum for logging
+                current_checksum = get_file_checksum(output_path) if os.path.exists(output_path) else None
+                log_download_attempt(version_dir, filename, "skipped", current_checksum, url)
+                # Update last_checked timestamp
+                update_version_info(version_file, filename, url, current_checksum, current_checksum, check_only=True)
+                return True, "skipped", f"File up to date: {filename} ({reason})"
 
         # Get current checksum if file exists
         old_checksum = None
@@ -159,6 +195,13 @@ def download_ontology_with_versioning(url, output_path, repo_path, force_downloa
         print(f"📥 Downloading {filename}...")
         response = download_with_retry(url)
 
+        # Collect remote metadata
+        remote_metadata = {
+            'remote_etag': response.headers.get('ETag', '').strip('"'),
+            'remote_size': response.headers.get('Content-Length'),
+            'remote_modified': response.headers.get('Last-Modified')
+        }
+        
         # Calculate new checksum
         new_checksum = get_file_checksum(response.content)
 
@@ -181,8 +224,9 @@ def __init__(self, content):
         else:
             handle_compressed_file(response, output_path, url)
 
-        # Update version tracking
-        update_version_info(version_file, filename, url, old_checksum, new_checksum)
+        # Update version tracking with remote metadata
+        update_version_info(version_file, filename, url, old_checksum, new_checksum, 
+                          remote_metadata=remote_metadata)
 
         # Log successful download
         status = "updated" if old_checksum else "new"
@@ -221,10 +265,18 @@ def download_ontology_safe(url, output_path, repo_path, force_download=False):
     )
 
     if success:
-        if status != "skipped":
+        if status == "skipped":
+            print(f"  ✓ Up-to-date: {filename}")
+        elif status == "no_change":
+            print(f"  ✓ No changes: {filename} (server version unchanged)")
+        elif status == "updated":
+            print(f"  ⟳ Updated: {filename}")
+        elif status == "new":
+            print(f"  ✅ Downloaded: {filename}")
+        else:
             print(f"   {message}")
     else:
-        print(f"❌ {message}")
+        print(f"  ❌ Failed: {filename} - {message}")
 
     return success
 

scripts/version_tracker.py

@@ -73,7 +73,7 @@ def log_download_attempt(version_dir, filename, status, checksum, url=None, erro
         f.write(log_entry + '\n')
 
 
-def update_version_info(version_file, filename, url, old_checksum, new_checksum):
+def update_version_info(version_file, filename, url, old_checksum, new_checksum, check_only=False, remote_metadata=None):
     """Update version tracking information."""
     version_info = load_version_info(version_file)
 
@@ -83,14 +83,25 @@ def update_version_info(version_file, filename, url, old_checksum, new_checksum)
         file_path = os.path.join(os.path.dirname(version_file), '..', filename)
         file_size = os.path.getsize(file_path)
 
-    version_info[filename] = {
-        'url': url,
-        'checksum': new_checksum,
-        'previous_checksum': old_checksum,
-        'last_updated': datetime.now().isoformat(),
-        'size_bytes': file_size,
-        'version_history': version_info.get(filename, {}).get('version_history', [])
-    }
+    # If we're just checking (not downloading), update last_checked only
+    if check_only and filename in version_info:
+        version_info[filename]['last_checked'] = datetime.now().isoformat()
+        # Update remote metadata if provided
+        if remote_metadata:
+            version_info[filename].update(remote_metadata)
+    else:
+        version_info[filename] = {
+            'url': url,
+            'checksum': new_checksum,
+            'previous_checksum': old_checksum,
+            'last_updated': datetime.now().isoformat(),
+            'last_checked': datetime.now().isoformat(),
+            'size_bytes': file_size,
+            'version_history': version_info.get(filename, {}).get('version_history', [])
+        }
+        # Add remote metadata if provided
+        if remote_metadata:
+            version_info[filename].update(remote_metadata)
 
     # Add to version history
     if old_checksum:
@@ -108,7 +119,7 @@ def get_version_status(version_file, filename):
     return version_info.get(filename, {})
 
 
-def should_download(filepath, url, version_file):
+def should_download(filepath, url, version_file, check_remote=True):
     """Determine if file should be downloaded based on version tracking."""
     filename = os.path.basename(filepath)
 
@@ -135,6 +146,13 @@ def should_download(filepath, url, version_file):
     if file_info.get('url') != url:
         return True, "url_changed"
 
+    # Check remote changes if enabled
+    if check_remote:
+        # Import here to avoid circular dependency
+        from enhanced_download import check_remote_changes
+        if check_remote_changes(url, file_info):
+            return True, "remote_changed"
+    
     return False, "up_to_date"