<Implemented Image Search and Alt Text generation>

.gitignore

@@ -5,6 +5,10 @@ __pycache__/
 
 # C extensions
 *.so
+.vs/
+*.vsidx
+*.wsuo
+*.sqlite
 
 # Distribution / packaging
 .Python

README.md

@@ -1,30 +1,24 @@
 # Moments
 
-A photo sharing social networking app built with Python and Flask. The example application for the book *[Python Web Development with Flask (2nd edition)](https://helloflask.com/en/book/4)* (《[Flask Web 开发实战（第 2 版）](https://helloflask.com/book/4)》).
+Moments is a photo-sharing social networking application developed using Python and the Flask web framework. The application allows users to share and explore photos within a social platform. We introduced two machine learning-powered features into the platform:
+1. Alternative Text Generation: Automatically generates alternative text (alt-text) for uploaded images using Azure Computer Vision API.
+2. Image Search: Uses object detection and image tagging to identify objects present in uploaded images. And enables keyword-based image search by allowing users to find images based on detected objects or tags.
 
-Demo: http://moments.helloflask.com
+## Instructions
+We are using Azure Computer Vision API for introducing the above ML features. Please follow the below steps to create the same -
+1. Sign in to Azure Portal
+2. Create a New Resource
+3. Configure the Resource by adding the Subscription, Resource Group, Region, Name and Pricing Tier
+4. Review & Create
+5. Once the resource is created, get API Key & Endpoint and update it in the code.
+6. Create a .env file to copy the API Key & Endpoint and set up the environment variables.
 
-![Screenshot](demo.png)
+Create a virtual environment using 'venv' and run the following commands
+1. Install dependencies with `pip install -r requirements.txt`
+2.  `pip install azure-ai-vision-imageanalysis`
+3.  `pip install azure-cognitiveservices-vision-computervision`
 
-## Installation
-
-Clone the repo:
-
-```
-$ git clone https://github.com/greyli/moments
-$ cd moments
-```
-
-Install dependencies with [PDM](https://pdm.fming.dev):
-
-```
-$ pdm install
-```
-
-> [!TIP]
-> If you don't have PDM installed, you can create a virtual environment with `venv` and install dependencies with `pip install -r requirements.txt`.
-
-To initialize the app, run the `flask init-app` command:
+Next, to initialize the app, run the `flask init-app` command:
 
 ```
 $ pdm run flask init-app
@@ -47,8 +41,3 @@ Now you can run the app:
 $ pdm run flask run
 * Running on http://127.0.0.1:5000/
 ```
-
-## License
-
-This project is licensed under the MIT License (see the
-[LICENSE](LICENSE) file for details).

app.py

@@ -11,3 +11,6 @@
 
 config_name = os.getenv('FLASK_CONFIG', 'development')
 app = create_app(config_name)
+
+AZURE_VISION_ENDPOINT = os.getenv("AZURE_VISION_ENDPOINT")
+AZURE_VISION_KEY = os.getenv("AZURE_VISION_KEY")

moments/blueprints/main.py

@@ -2,13 +2,13 @@
 from flask_login import current_user, login_required
 from sqlalchemy import func, select
 from sqlalchemy.orm import with_parent
-
+import os
 from moments.core.extensions import db
 from moments.decorators import confirm_required, permission_required
 from moments.forms.main import CommentForm, DescriptionForm, TagForm
-from moments.models import Collection, Comment, Follow, Notification, Photo, Tag, User
+from moments.models import Collection, Comment, Follow, Notification, Photo, Tag, User, photo_tag
 from moments.notifications import push_collect_notification, push_comment_notification
-from moments.utils import flash_errors, redirect_back, rename_image, resize_image, validate_image
+from moments.utils import flash_errors, redirect_back, rename_image, resize_image, validate_image, imageAltTextGeneration, analyzeImage
 
 main_bp = Blueprint('main', __name__)
 
@@ -130,14 +130,37 @@ def upload():
         if not validate_image(f.filename):
             return 'Invalid image.', 400
         filename = rename_image(f.filename)
-        f.save(current_app.config['MOMENTS_UPLOAD_PATH'] / filename)
+        image_path = current_app.config['MOMENTS_UPLOAD_PATH']/filename
+        f.save(str(image_path))
         filename_s = resize_image(f, filename, current_app.config['MOMENTS_PHOTO_SIZES']['small'])
         filename_m = resize_image(f, filename, current_app.config['MOMENTS_PHOTO_SIZES']['medium'])
+        f.seek(0)
+        binaryImage = f.read()
+        description = imageAltTextGeneration(binaryImage)
+
         photo = Photo(
-            filename=filename, filename_s=filename_s, filename_m=filename_m, author=current_user._get_current_object()
+            filename=filename, 
+            filename_s=filename_s, 
+            filename_m=filename_m, 
+            author=current_user._get_current_object(), 
+            description=description
         )
         db.session.add(photo)
         db.session.commit()
+        f.seek(0)
+        binaryImage = f.read()
+        tags = analyzeImage(binaryImage)
+        tag_objects = []
+        for tag_name in tags:
+            tag = db.session.scalar(select(Tag).filter_by(name=tag_name))
+            if tag is None:
+                tag = Tag(name=tag_name)
+                db.session.add(tag)
+                db.session.commit()
+            tag_objects.append(tag)
+        photo.tags.extend(tag_objects)  
+        db.session.commit()
+
     return render_template('main/upload.html')
 
 

moments/templates/main/search.html

@@ -6,41 +6,78 @@
 
 {% block content %}
 <div class="page-header">
-  <h1>Search: {{ q }}</h1>
+  <h1>Search: "{{ q }}"</h1>
 </div>
+
 <div class="row">
+
   <div class="col-md-3">
     <div class="nav nav-pills flex-column" role="tablist" aria-orientation="vertical">
       <a class="nav-item nav-link {% if category == 'photo' %}active{% endif %}"
-        href="{{ url_for('.search', q=q, category='photo') }}">Photo</a>
+        href="{{ url_for('.search', q=q, category='photo') }}">Photos</a>
       <a class="nav-item nav-link {% if category == 'user' %}active{% endif %}"
-        href="{{ url_for('.search', q=q, category='user') }}">User</a>
+        href="{{ url_for('.search', q=q, category='user') }}">Users</a>
       <a class="nav-item nav-link {% if category == 'tag' %}active{% endif %}"
-        href="{{ url_for('.search', q=q, category='tag') }}">Tag</a>
+        href="{{ url_for('.search', q=q, category='tag') }}">Tags</a>
     </div>
   </div>
+
+
   <div class="col-md-9">
     {% if results %}
-      <h5>{{ results|length }} results</h5>
-      {% for item in results %}
+      <h5 class="mb-3">{{ results|length }} results found</h5>
+
       {% if category == 'photo' %}
-        {{ photo_card(item) }}
+        <div class="row row-cols-1 row-cols-md-3 g-4">
+          {% for photo in results %}
+            <div class="col">
+              <div class="card">
+                <a href="{{ url_for('main.show_photo', photo_id=photo.id) }}">
+                  <img src="{{ url_for('main.get_image', filename=photo.filename_m) }}" class="card-img-top" alt="Photo">
+                </a>
+                <div class="card-body">
+                  <h5 class="card-title">{{ photo.description or "No description" }}</h5>
+                  <p class="card-text">
+                    Tags: 
+                    {% for tag in photo.tags %}
+                      <a href="{{ url_for('.search', q=tag.name, category='tag') }}" class="badge bg-secondary">
+                        #{{ tag.name }}
+                      </a>
+                    {% endfor %}
+                  </p>
+                  <a href="{{ url_for('main.show_photo', photo_id=photo.id) }}" class="btn btn-primary">View</a>
+                </div>
+              </div>
+            </div>
+          {% endfor %}
+        </div>
       {% elif category == 'user' %}
-        {{ user_card(item) }}
-      {% else %}
-      <a class="badge text-bg-light rounded-pill" href="{{ url_for('.show_tag', tag_id=item.id) }}">
-        {{ item.name }} {{ item.photos|length }}
-      </a>
+        {% for user in results %}
+          <div class="mb-3">
+            {{ user_card(user) }}
+          </div>
+        {% endfor %}
+      {% elif category == 'tag' %}
+        <div class="d-flex flex-wrap">
+          {% for tag in results %}
+            <a href="{{ url_for('.search', q=tag.name, category='photo') }}" class="badge bg-primary m-1 p-2">
+              #{{ tag.name }} ({{ tag.photos_count }} photos)
+            </a>
+          {% endfor %}
+        </div>
       {% endif %}
-      {% endfor %}
+
     {% else %}
-    <h5 class="tip">No results.</h5>
+      <h5 class="tip">No results found for "{{ q }}".</h5>
     {% endif %}
   </div>
 </div>
+
+<!-- Pagination -->
 {% if results %}
-<div class="page-footer">
-  {{ render_pagination(pagination, align='right') }}
-</div>
+  <div class="page-footer mt-4">
+    {{ render_pagination(pagination, align='right') }}
+  </div>
 {% endif %}
+
 {% endblock %}

moments/utils.py

@@ -8,6 +8,11 @@
 from flask import current_app, flash, redirect, request, url_for
 from jwt.exceptions import InvalidTokenError
 from PIL import Image
+from azure.ai.vision.imageanalysis import ImageAnalysisClient
+from azure.core.credentials import AzureKeyCredential
+from azure.ai.vision.imageanalysis.models import VisualFeatures
+from msrest.authentication import CognitiveServicesCredentials
+import os
 
 
 def generate_token(user, operation, expiration=3600, **kwargs):
@@ -76,3 +81,34 @@ def flash_errors(form):
     for field, errors in form.errors.items():
         for error in errors:
             flash(f'Error in the {getattr(form, field).label.text} field - {error}')
+
+def imageAltTextGeneration(image):
+
+    endpoint=os.getenv("AZURE_VISION_ENDPOINT")
+    credentials=AzureKeyCredential(os.getenv("AZURE_VISION_KEY"))
+    client = ImageAnalysisClient(
+        endpoint=endpoint,
+        credential=credentials
+        )
+
+    response = client.analyze(
+        image_data=image,  
+        visual_features=[VisualFeatures.CAPTION]  
+    )
+    return response.caption.text if response.caption else "No caption available."
+
+
+def analyzeImage(imageBinary : bytes):
+    endpoint=os.getenv("AZURE_VISION_ENDPOINT")
+    credentials=AzureKeyCredential(os.getenv("AZURE_VISION_KEY"))
+
+    client = ImageAnalysisClient(
+        endpoint=endpoint,
+        credential=credentials
+        )
+    response = client.analyze(
+        image_data=imageBinary,
+        visual_features=[VisualFeatures.TAGS]
+    )
+
+    return [value.get("name", "") for value in response.get("tagsResult", {}).get("values", [])]