feat: Instaloader scraping, entity extraction, and user management

README.md

@@ -0,0 +1,113 @@
+# Friendly 🤝
+
+**Automatically find friends within your close network with shared interests.**
+
+Skip the awkward conversations and get straight into deep conversations about shared passions that go deeper than "how was your day?"
+
+## How It Works
+
+1. **Connect** — Sync your Instagram account
+2. **Analyze** — AI analyzes your posts, captions, and bio to map your interests
+3. **Discover** — See a visual graph of your interests and find people who share them
+4. **Connect** — Get AI-generated icebreakers for shared interests
+
+## Tech Stack
+
+### Frontend
+
+- **Next.js** + React + TypeScript
+- **Tailwind CSS** + shadcn/ui
+- **D3.js** force-directed graph visualization
+- **Bun** runtime
+
+### Backend
+
+- **FastAPI** (Python)
+- **Neo4j** graph database
+- **Instaloader** Instagram scraping
+- **UV** package manager
+
+### AI Services
+
+| Service      | Role                                                       |
+| ------------ | ---------------------------------------------------------- |
+| **Reka**     | Image analysis, interest extraction, icebreaker generation |
+| **Yutori**   | Research & scouting tasks for interest enrichment          |
+| **Modulate** | Speech-to-text for voice ingestion                         |
+
+## Getting Started
+
+### Prerequisites
+
+- Node.js 18+ / Bun
+- Python 3.11+
+- Docker (for Neo4j)
+
+### Setup
+
+```bash
+# Clone
+git clone https://github.com/shlawgathon/friendly.git
+cd friendly
+
+# Backend
+cd backend
+cp .env.example .env  # Add your API keys
+uv sync
+source .venv/bin/activate
+uvicorn app.main:app --port 8000 --reload
+
+# Frontend (new terminal)
+cd frontend
+bun install
+bun run dev
+```
+
+### Neo4j
+
+```bash
+# Local (Docker)
+docker compose up neo4j -d
+
+# Or use Neo4j Aura (cloud) — update NEO4J_URI in .env
+```
+
+### Environment Variables
+
+```env
+# Neo4j
+NEO4J_URI=bolt://localhost:7687
+NEO4J_USER=neo4j
+NEO4J_PASSWORD=your_password
+
+# AI Services
+REKA_API_KEY=your_key
+YUTORI_API_KEY=your_key
+MODULATE_API_KEY=your_key
+PIONEER_API_KEY=your_key
+```
+
+## Architecture
+
+```
+┌─────────────┐     ┌─────────────┐     ┌─────────────┐
+│   Frontend   │────▶│   Backend   │────▶│   Neo4j     │
+│  Next.js     │     │   FastAPI   │     │   Graph DB  │
+└─────────────┘     └──────┬──────┘     └─────────────┘
+                           │
+                    ┌──────┼──────┐
+                    ▼      ▼      ▼
+                  Reka  Yutori  Modulate
+```
+
+### Ingestion Pipeline
+
+1. **Scrape** — Instaloader fetches profile, posts, and carousel images
+2. **Analyze** — Reka vision analyzes each image with post caption context
+3. **Extract** — Reka extracts structured interests (hobbies, brands) from combined text
+4. **Store** — Entities written to Neo4j as `User → INTERESTED_IN → Hobby` / `User → FOLLOWS → Brand`
+5. **Enrich** — Yutori submits research & scouting tasks for top interests
+
+## License
+
+MIT

backend/.gitignore

@@ -0,0 +1,6 @@
+.env
+__pycache__/
+*.pyc
+.venv/
+*.egg-info/
+dist/

backend/app/config.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    # ── Neo4j ──
+    neo4j_uri: str = "bolt://localhost:7687"
+    neo4j_user: str = "neo4j"
+    neo4j_password: str = "friendly_dev_password"
+
+    # ── Sponsor API Keys ──
+    modulate_api_key: str = ""
+    reka_api_key: str = ""
+    pioneer_api_key: str = ""
+    yutori_api_key: str = ""
+
+
+    # ── Backend ──
+    backend_host: str = "0.0.0.0"
+    backend_port: int = 8000
+    webhook_base_url: str = "http://localhost:8000"
+
+    # ── Hard Caps ──
+    max_posts_per_ingest: int = 10
+    max_posts_hard_limit: int = 25
+    top_interests_for_yutori: int = 3
+    max_parallel_reka_calls: int = 2
+
+    # ── Retry / Timeouts ──
+    api_timeout_seconds: float = 20.0
+    max_retries: int = 3
+    retry_backoff_multiplier: float = 1.0
+    retry_backoff_max: float = 30.0
+
+    # ── Cooldowns ──
+    ingest_cooldown_minutes: int = 5
+
+
+settings = Settings()

backend/app/db/neo4j.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import logging
+from contextlib import asynccontextmanager
+from typing import AsyncIterator
+
+from neo4j import AsyncGraphDatabase, AsyncDriver, AsyncSession
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+_driver: AsyncDriver | None = None
+
+
+async def get_driver() -> AsyncDriver:
+    global _driver
+    if _driver is None:
+        _driver = AsyncGraphDatabase.driver(
+            settings.neo4j_uri,
+            auth=(settings.neo4j_user, settings.neo4j_password),
+        )
+        await _driver.verify_connectivity()
+        logger.info("Neo4j connected: %s", settings.neo4j_uri)
+    return _driver
+
+
+async def close_driver() -> None:
+    global _driver
+    if _driver:
+        await _driver.close()
+        _driver = None
+        logger.info("Neo4j driver closed")
+
+
+@asynccontextmanager
+async def get_session() -> AsyncIterator[AsyncSession]:
+    driver = await get_driver()
+    async with driver.session() as session:
+        yield session

backend/app/db/schema.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import logging
+
+from app.db.neo4j import get_session
+
+logger = logging.getLogger(__name__)
+
+SCHEMA_QUERIES = [
+    # Constraints
+    "CREATE CONSTRAINT user_id IF NOT EXISTS FOR (u:User) REQUIRE u.id IS UNIQUE",
+    "CREATE CONSTRAINT hobby_name IF NOT EXISTS FOR (h:Hobby) REQUIRE h.name IS UNIQUE",
+    "CREATE CONSTRAINT location_name IF NOT EXISTS FOR (l:Location) REQUIRE l.name IS UNIQUE",
+    "CREATE CONSTRAINT brand_name IF NOT EXISTS FOR (b:Brand) REQUIRE b.name IS UNIQUE",
+    "CREATE CONSTRAINT activity_name IF NOT EXISTS FOR (a:Activity) REQUIRE a.name IS UNIQUE",
+    "CREATE CONSTRAINT task_record_id IF NOT EXISTS FOR (t:TaskRecord) REQUIRE t.provider_task_id IS UNIQUE",
+    "CREATE CONSTRAINT ingest_job_id IF NOT EXISTS FOR (j:IngestJob) REQUIRE j.job_id IS UNIQUE",
+    # Indexes
+    "CREATE INDEX user_username IF NOT EXISTS FOR (u:User) ON (u.username)",
+    "CREATE INDEX ingest_job_status IF NOT EXISTS FOR (j:IngestJob) ON (j.status)",
+    "CREATE INDEX task_record_status IF NOT EXISTS FOR (t:TaskRecord) ON (t.status)",
+]
+
+
+async def init_schema() -> None:
+    """Create constraints and indexes if they don't exist."""
+    async with get_session() as session:
+        for query in SCHEMA_QUERIES:
+            try:
+                await session.run(query)
+            except Exception as e:
+                # Constraint already exists — safe to ignore
+                logger.debug("Schema query skipped: %s", e)
+    logger.info("Neo4j schema initialized (%d queries)", len(SCHEMA_QUERIES))

backend/app/main.py

@@ -0,0 +1,67 @@
+"""Friendly backend — FastAPI application."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.db.neo4j import get_driver, close_driver
+from app.db.schema import init_schema
+from app.routers import ingest, jobs, discover, chat, webhooks
+from app.workers.yutori_poller import start_poller
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(name)s — %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    logger.info("Starting Friendly backend...")
+    await get_driver()
+    await init_schema()
+
+    # Start Yutori poller background task
+    poller_task = asyncio.create_task(start_poller())
+    logger.info("Backend ready")
+
+    yield
+
+    # Shutdown
+    poller_task.cancel()
+    await close_driver()
+    logger.info("Backend shutdown complete")
+
+
+app = FastAPI(
+    title="Friendly",
+    description="Semantic social graph — find friends through shared passions",
+    version="0.1.0",
+    lifespan=lifespan,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:3000", "http://localhost:3001"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Register routers
+app.include_router(ingest.router)
+app.include_router(jobs.router)
+app.include_router(discover.router)
+app.include_router(chat.router)
+app.include_router(webhooks.router)
+
+
+@app.get("/health")
+async def health():
+    return {"status": "ok", "service": "friendly-backend"}