diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b5da7e7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +.git +.github +.venv +venv +env +ocrenv +__pycache__ +**/__pycache__ +.pytest_cache +.mypy_cache +.coverage +htmlcov +.env +*.pyc +*.pyo +.idea +.vscode +tests +README.md +PRODUCTIONIZING.md diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b50230d --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Copy this file to .env and fill in real values. .env is gitignored. +# Leave the values empty to boot the UI and API anyway: text analysis +# returns a clear "credentials are not configured" message, and OCR still runs. + +# Endpoint of an Azure AI Language resource, without a path. +# Example: https://my-language.cognitiveservices.azure.com +AZURE_LANGUAGE_ENDPOINT= + +# Key for that same resource (Keys and Endpoint blade in the Azure portal). +AZURE_LANGUAGE_KEY= + +# Optional. The Streamlit service uses this to find the API. +# Docker Compose sets it to http://api:8000 for the ui container. +# A local `streamlit run` defaults to http://localhost:8000 when this is unset. +# API_BASE_URL=http://localhost:8000 diff --git a/.github/workflows/main_text-analytics-fastapi.yml b/.github/workflows/main_text-analytics-fastapi.yml index d0a4ed9..964aaa2 100644 --- a/.github/workflows/main_text-analytics-fastapi.yml +++ b/.github/workflows/main_text-analytics-fastapi.yml @@ -1,6 +1,7 @@ -# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy -# More GitHub Actions for Azure: https://github.com/Azure/actions -# More info on Python, GitHub Actions, and Azure App Service: https://aka.ms/python-webapps-actions +# Deploy the API (not the Streamlit UI) to the existing Azure App Service. +# Text analysis stays dark until the App Service has AZURE_LANGUAGE_ENDPOINT +# and AZURE_LANGUAGE_KEY configured. See PRODUCTIONIZING.md for the Container +# Apps, Key Vault, and identity-based setup that should replace this file. name: Build and deploy Python app to Azure Web App - text-analytics-fastapi @@ -15,49 +16,50 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python version - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: "3.12" + + - name: Install Tesseract + run: sudo apt-get update && sudo apt-get install -y tesseract-ocr - - name: Create and start virtual environment - run: | - python -m venv venv - source venv/bin/activate - - name: Install dependencies - run: pip install -r requirements.txt - - # Optional: Add step to run tests here (PyTest, Django test suites, etc.) - + run: pip install -r requirements-dev.txt + + - name: Run tests + run: pytest + - name: Upload artifact for deployment jobs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: python-app path: | - . + . + !.git/ + !.venv/ !venv/ deploy: runs-on: ubuntu-latest needs: build environment: - name: 'Production' + name: "Production" url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} steps: - name: Download artifact from build job - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: python-app path: . - - - name: 'Deploy to Azure Web App' - uses: azure/webapps-deploy@v2 + + - name: Deploy to Azure Web App + uses: azure/webapps-deploy@v3 id: deploy-to-webapp with: - app-name: 'text-analytics-fastapi' - slot-name: 'Production' + app-name: "text-analytics-fastapi" + slot-name: "Production" publish-profile: ${{ secrets.AZUREAPPSERVICE_PUBLISHPROFILE_F64BD7EE84E649C1930A5AA58F25A0D0 }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..8aec6ed --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,56 @@ +name: Tests + +on: + push: + pull_request: + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Tesseract + run: sudo apt-get update && sudo apt-get install -y tesseract-ocr + + - name: Install Python dependencies + run: pip install -r requirements-dev.txt + + - name: Run tests + run: pytest + + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build images + run: docker compose build + + - name: Boot the stack and smoke-test it + run: | + docker compose up -d + for _ in $(seq 1 30); do + curl -fsS http://localhost:8000/health && break + sleep 2 + done + curl -fsS http://localhost:8000/health + code=$(curl -s -o /tmp/analyze.json -w "%{http_code}" \ + -X POST http://localhost:8000/api/v1/analyze \ + -H 'content-type: application/json' \ + -d '{"text_to_analyze":["hello"]}') + test "$code" = "503" + grep -q "not configured" /tmp/analyze.json + curl -fsS -F "images=@samples/ocr-sample.png" \ + http://localhost:8000/api/v1/extract_text \ + | grep -i -q "quick brown" + for _ in $(seq 1 30); do + curl -fsS http://localhost:8501/_stcore/health && break + sleep 2 + done + curl -fsS http://localhost:8501/_stcore/health + docker compose down diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 0000000..0ee4905 --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,22 @@ +# Light page, dark text. Loaded when Streamlit is started from the repo root +# and when the frontend image uses /app as its working directory. + +[theme] +base = "light" +primaryColor = "#0f4c81" +backgroundColor = "#f6f3ee" +secondaryBackgroundColor = "#ffffff" +textColor = "#1c1917" +font = "sans serif" + +[server] +headless = true +address = "0.0.0.0" +# Matches the API limit in app/ocr.py. Streamlit's default is 200 MB. +maxUploadSize = 10 + +[browser] +gatherUsageStats = false + +[client] +toolbarMode = "minimal" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fcb853a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# API image: FastAPI plus the Tesseract binary OCR needs. +# Build from the repository root: docker build -t text-ocr-api . + +FROM python:3.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends tesseract-ocr \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY main.py . +COPY text-analytics-sdk ./text-analytics-sdk + +RUN useradd --create-home --uid 10001 appuser \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health')" + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Dockerfile.frontend b/Dockerfile.frontend new file mode 100644 index 0000000..c9d7030 --- /dev/null +++ b/Dockerfile.frontend @@ -0,0 +1,29 @@ +# Streamlit UI. It only needs the frontend package and a URL for the API. +# Build from the repository root: docker build -f Dockerfile.frontend -t text-ocr-ui . + +FROM python:3.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + STREAMLIT_BROWSER_GATHER_USAGE_STATS=false \ + API_BASE_URL=http://api:8000 + +WORKDIR /app + +COPY requirements-ui.txt . +RUN pip install --no-cache-dir -r requirements-ui.txt + +COPY frontend ./frontend +COPY .streamlit ./.streamlit + +RUN useradd --create-home --uid 10001 appuser \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8501 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8501/_stcore/health')" + +CMD ["streamlit", "run", "frontend/streamlit_app.py", "--server.address", "0.0.0.0", "--server.port", "8501", "--server.headless", "true"] diff --git a/PRODUCTIONIZING.md b/PRODUCTIONIZING.md new file mode 100644 index 0000000..78d19df --- /dev/null +++ b/PRODUCTIONIZING.md @@ -0,0 +1,321 @@ +# Productionizing the Text & OCR Workbench + +The Docker Compose stack is a local demo. It has no login, no secret store, and no private network path to Azure. This document is the concrete path from that demo to a deployment you can leave running. Do the steps in order. Each one removes a specific risk the current tree still has. + +```mermaid +flowchart TB + dev["Developer"] + gha["GitHub Actions
OIDC, no publish profile"] + acr["Azure Container Registry"] + ca_ui["Container Apps
Streamlit"] + ca_api["Container Apps
FastAPI"] + entra["Microsoft Entra ID"] + kv["Key Vault
RBAC + managed identity"] + lang["Azure AI Language
private endpoint"] + ai["Application Insights
OpenTelemetry"] + law["Log Analytics"] + + dev -->|"pull request"| gha + gha -->|"pytest, image build, scan"| acr + acr --> ca_ui + acr --> ca_api + entra -->|"Easy Auth"| ca_ui + entra -->|"bearer tokens"| ca_api + ca_ui -->|"HTTP inside the Container Apps environment"| ca_api + ca_api -->|"Key Vault reference"| kv + ca_api --> lang + ca_api --> ai + ca_ui --> ai + ai --> law +``` + +## 1. Rotate the credentials that were committed + +`main.py` used to contain a live Language subscription key and an Application Insights instrumentation key. `utils.py` used to call `https://language-project.cognitiveservices.azure.com`. Those values are still in git history even though they are gone from the current tree. Anyone who can read the repository can read the old blobs. + +Treat both secrets as compromised: + +1. In the Azure portal, open the Language resource whose endpoint is `language-project.cognitiveservices.azure.com` (or whichever resource that host still points at). +2. On **Keys and Endpoint**, regenerate **Key 1** and **Key 2**. +3. Open the Application Insights resource that received logs from this app and rotate its connection string. If you cannot tell which resource it was, create a new Application Insights resource and stop using the old one. +4. Do not paste the old key into `.env`, App Service settings, or a GitHub secret "just to get it working again." + +After rotation, the existing App Service (`text-analytics-fastapi`) will fail Azure calls until step 2 below is done. That is the safe failure. The API returns HTTP 502 for a rejected key and HTTP 503 when the settings are empty. It does not fall back to a baked-in key. + +## 2. Give the current App Service a secret without committing one + +The workflow `.github/workflows/main_text-analytics-fastapi.yml` deploys the API to Azure App Service on every push to `main`. It uses a publish profile stored as a GitHub secret. Keep that workflow only long enough to move off it. While it is still the deploy path, point the app at Key Vault instead of pasting keys into configuration. + +Create a vault and a managed identity the app can use: + +```bash +az group create --name rg-text-ocr --location eastus +az keyvault create \ + --name kv-text-ocr- \ + --resource-group rg-text-ocr \ + --location eastus \ + --enable-rbac-authorization true + +az cognitiveservices account keys list \ + --name \ + --resource-group +``` + +Store only the new key: + +```bash +az keyvault secret set \ + --vault-name kv-text-ocr- \ + --name azure-language-key \ + --value "" +``` + +Turn on a system-assigned identity for the existing app and grant it read access to secrets: + +```bash +az webapp identity assign \ + --name text-analytics-fastapi \ + --resource-group + +# principalId is in the JSON from the command above +az role assignment create \ + --role "Key Vault Secrets User" \ + --assignee-object-id \ + --assignee-principal-type ServicePrincipal \ + --scope $(az keyvault show --name kv-text-ocr- --query id -o tsv) +``` + +App Service application settings, not the repo: + +| Name | Value | +| --- | --- | +| `AZURE_LANGUAGE_ENDPOINT` | `https://.cognitiveservices.azure.com` | +| `AZURE_LANGUAGE_KEY` | `@Microsoft.KeyVault(VaultName=kv-text-ocr-;SecretName=azure-language-key)` | + +The endpoint is not a secret. The key reference resolves at runtime through the managed identity. Confirm the setting shows a green Key Vault reference in the portal, then hit `POST /api/v1/analyze` and expect HTTP 200. + +The App Service image is a code deploy, not the Docker image, so the `tesseract` binary is not there unless you add it. OCR will answer HTTP 503 with the install message until you move to the container in step 4. That message is the correct behavior. Do not apt-get over SSH on the app as a permanent fix. + +Startup command stays: + +```bash +gunicorn -w 2 -k uvicorn.workers.UvicornWorker main:app +``` + +## 3. Close the anonymous API before you share the URL + +`text-analytics-fastapi` is unauthenticated. Anyone who can reach it can spend your Language quota and upload files for OCR. Put Microsoft Entra ID in front of it before sharing the host. + +On the App Service: **Authentication** → add an identity provider → Microsoft → require authentication for unauthenticated requests. Use a new app registration, for example `text-ocr-api`. Restrict the app registration to accounts in your tenant. + +For the later Container Apps deploy, use the same app registration: + +- Streamlit ingress: Container Apps **Easy Auth** with that app registration, so a browser without a tenant login never loads the workbench. +- API ingress: require a bearer token. Set the API container to accept tokens whose audience is the API's application ID URI, `api://`. +- Streamlit, running server-side, fetches a token with the user's session (Easy Auth forwards `X-MS-TOKEN-AAD-ACCESS-TOKEN`) and sends `Authorization: Bearer` to the API. Reject requests that arrive without it. Add that check in `app/main.py` behind an env flag such as `AUTH_REQUIRED=true` so the local Compose stack can keep running without Entra. + +Service-to-service callers, if you add any later, get their own app registration and a client-credentials token. Do not ship a long-lived API key in a header as the primary control. + +## 4. Build images in GitHub Actions and run them on Container Apps + +The current workflow uploads the source tree and asks App Service to `pip install` it. Replace that with images you have already tested, pushed to Azure Container Registry by a workflow that does not store a password. + +### Identity for GitHub + +```bash +az identity create --name id-gha-text-ocr --resource-group rg-text-ocr +az acr create --name acrtextocr --resource-group rg-text-ocr --sku Basic + +# Federated credential for pushes to main. GitHub exchanges an OIDC token +# for this identity, so the workflow does not store a client secret. +az identity federated-credential create \ + --name gha-main \ + --identity-name id-gha-text-ocr \ + --resource-group rg-text-ocr \ + --issuer https://token.actions.githubusercontent.com \ + --subject repo:ovokpus/Python-Azure-AI-REST-APIs:ref:refs/heads/main \ + --audiences api://AzureADTokenExchange + +az role assignment create \ + --role AcrPush \ + --assignee-object-id $(az identity show -n id-gha-text-ocr -g rg-text-ocr --query principalId -o tsv) \ + --assignee-principal-type ServicePrincipal \ + --scope $(az acr show -n acrtextocr --query id -o tsv) +``` + +### Workflow shape + +Replace `.github/workflows/main_text-analytics-fastapi.yml` with a workflow that: + +1. Runs `pytest` on the pull request (`.github/workflows/test.yml` already does this, including a Compose boot). +2. On `main` only, logs in with `azure/login@v2` using `client-id`, `tenant-id`, and `subscription-id` — no client secret. +3. Runs `az acr login` and builds both Dockerfiles. +4. Pushes `text-ocr-api:` and `text-ocr-ui:`. +5. Deploys those tags to Container Apps. +6. Deletes the publish-profile GitHub secret after one green deploy. A publish profile is a password for the whole app. + +A minimal deploy job after the images exist: + +```yaml +permissions: + id-token: write + contents: read + +steps: + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - uses: azure/container-apps-deploy-action@v2 + with: + resourceGroup: rg-text-ocr + containerAppName: ca-text-ocr-api + imageToDeploy: acrtextocr.azurecr.io/text-ocr-api:${{ github.sha }} +``` + +`AZURE_CLIENT_ID` here is the client id of the user-assigned identity, not a secret. The federated credential is what makes the OIDC token valid. Do not also create a client secret "as a backup." + +### Container Apps environment + +One environment, two apps, one internal API: + +```bash +az containerapp env create \ + --name cae-text-ocr \ + --resource-group rg-text-ocr \ + --location eastus + +az containerapp create \ + --name ca-text-ocr-api \ + --resource-group rg-text-ocr \ + --environment cae-text-ocr \ + --image acrtextocr.azurecr.io/text-ocr-api:latest \ + --target-port 8000 \ + --ingress internal \ + --min-replicas 1 \ + --max-replicas 3 \ + --cpu 0.5 --memory 1.0Gi \ + --system-assigned + +az containerapp create \ + --name ca-text-ocr-ui \ + --resource-group rg-text-ocr \ + --environment cae-text-ocr \ + --image acrtextocr.azurecr.io/text-ocr-ui:latest \ + --target-port 8501 \ + --ingress external \ + --min-replicas 1 \ + --max-replicas 2 \ + --env-vars API_BASE_URL=https://ca-text-ocr-api.internal. +``` + +Probe both apps with the paths the Dockerfiles already use: + +| App | Path | Port | +| --- | --- | --- | +| API | `/health` | 8000 | +| UI | `/_stcore/health` | 8501 | + +`/health` is liveness only. It stays 200 when Azure credentials are missing, so a bad Key Vault reference will not crash-loop the revision. Watch `/api/v1/status` from a scheduled check (or an alert on HTTP 503 from `/api/v1/analyze`) so a missing secret is visible. + +Give `ca-text-ocr-api` a system-assigned identity, the **Key Vault Secrets User** role from step 2, and secret refs: + +```bash +az containerapp secret set \ + --name ca-text-ocr-api \ + --resource-group rg-text-ocr \ + --secrets language-key=keyvaultref:https://kv-text-ocr-.vault.azure.net/secrets/azure-language-key,identityref:system + +az containerapp update \ + --name ca-text-ocr-api \ + --resource-group rg-text-ocr \ + --set-env-vars AZURE_LANGUAGE_ENDPOINT=https://.cognitiveservices.azure.com \ + --replace-env-vars AZURE_LANGUAGE_KEY=secretref:language-key +``` + +Set the UI's `API_BASE_URL` to the API's internal ingress URL. Do not publish the API on a public port once the UI is the only caller. + +The API image already runs as uid 10001 and already installs `tesseract-ocr`. Keep that. Do not switch the container to root to "make OCR easier." + +## 5. Lock the Language resource to the virtual network + +A key in Key Vault still works from any network that has the key. After Container Apps is up: + +1. Create a virtual network and a private endpoint for the Language resource. +2. Disable public network access on the Cognitive Services account. +3. Put the Container Apps environment in that virtual network (workload profiles environment, or a regional environment with VNet injection). +4. Confirm `POST /api/v1/analyze` succeeds from the container and fails from a laptop that is not on the VPN. The laptop should keep using Compose with a dev Language resource, not the production key. + +## 6. Observability that does not store your users' text + +The demo logs to stdout: request method, path, status, and how many documents were analyzed. It does not log document text or the subscription key. Keep that rule in production. Hotel reviews and uploaded photos are customer content. + +Wire OpenTelemetry to Application Insights. Install these in the API image when you take this step: + +```text +azure-monitor-opentelemetry +opentelemetry-instrumentation-fastapi +``` + +Set one environment variable on both container apps, from Key Vault the same way as the Language key: + +```bash +APPLICATIONINSIGHTS_CONNECTION_STRING=secretref:appinsights-connection-string +``` + +Instrument in `app/main.py` only when that variable is present, so local Compose stays quiet: + +```python +import os + +if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): + from azure.monitor.opentelemetry import configure_azure_monitor + + configure_azure_monitor() +``` + +Add these alerts on the Application Insights resource: + +| Signal | Why | +| --- | --- | +| HTTP 5xx rate on the API above a handful in 5 minutes | Azure outage or a bug in a new image | +| HTTP 401/403 from the dependency call to `*.cognitiveservices.azure.com` | The Key Vault secret is the wrong key, or it was rotated and the reference is stale | +| CPU above 80% for 10 minutes on the API | Tesseract is burning a core per request; raise replicas or move OCR to Azure AI Vision | +| Language resource throttling (429) | Quota. Back off in `app/text_analytics.py` and request a quota increase | + +Dependency tracking shows the outbound call to Language. Confirm a test request produces a dependency span and that the span's custom properties do not include the raw text. If you want a business event ("document analyzed"), log the sentiment label and the document length, not the document. + +The UI container's stdout should be collected by the same Log Analytics workspace Container Apps uses by default. You do not need a second agent. + +## 7. OCR at a scale Tesseract will not enjoy + +Tesseract inside the API container is the right call for the demo and for light use. It has a hard ceiling: each request holds a CPU, there is no queue, and a 10 MB upload limit is the only guard. When that shows up in the CPU alert above, switch OCR to **Azure AI Vision Read** and keep Tesseract as the offline fallback behind a setting such as `OCR_BACKEND=vision|tesseract`. + +That client should use the same managed identity (`DefaultAzureCredential`) against a Vision resource, not a second subscription key in the environment. The 503 path you already have is how the API should behave when Vision is not configured. + +## 8. Hardening checklist before you call it done + +These are small, and each one is still open: + +- **Upload limits.** 10 MB per image is enforced in `app/ocr.py`. Add an ingress request-size limit on Container Apps to match, so the platform rejects a huge body before Python reads it. +- **Document content.** Text is forwarded to Azure Language, which is a processor under your Azure subscription. Say that on the Streamlit page before you let anyone outside the team use it. Do not write the text to disk. The current OCR path keeps the image in memory and does not save uploads into the repo (the old `fastapi-app` helper did). +- **Dependency scanning.** Enable Dependabot or `pip-audit` in the test workflow for `requirements.txt` and `requirements-ui.txt`. The pins are there so a scan has something stable to diff. +- **Image scanning.** After the ACR push, run `az acr task` with a scanner, or Microsoft Defender for Containers, and block deploy on high-severity findings. +- **CORS.** Leave it off. The browser talks to Streamlit, and Streamlit talks to the API server-side. A wide-open `CORSMiddleware` would undo the ingress split. +- **Replicas and the key.** Scale the API horizontally. The app holds no session state. Do not enable sticky sessions to paper over a bug. +- **Local `.env`.** Keep `.env` gitignored. Add a CI step that fails if a file named `.env` is tracked (`git ls-files .env`). +- **History.** Rotating the keys (step 1) matters more than rewriting git history. Rewriting history on `main` does not revoke the key. Do step 1 even if you never rewrite. + +## What is already in place + +You do not need to rebuild these for the demo: + +- Graceful startup when Azure credentials are absent (HTTP 503 plus a message that names the variables). +- Placeholder values from `.env.example` treated as unset, so a copied template cannot accidentally call Azure. +- Health checks in both Dockerfiles and in `docker-compose.yml`. +- Non-root user in both images. +- Tests for the credential failure, a mocked Azure success, a rejected key, and OCR. +- A pull-request workflow that builds the images and smoke-tests them. + +Production work starts at the key rotation, not at another rewrite of the UI. diff --git a/README.md b/README.md index fdedb75..712a113 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,275 @@

Braided cables plugged into brass sockets on a dark patch panel

-# Python-Azure-AI-REST-APIs +# Text & OCR Workbench + +A small FastAPI service and a Streamlit UI for two jobs: score sentiment and pull key phrases with **Azure AI Language**, and read text out of images with **Tesseract**. Both run on your laptop with Docker Compose. Azure keys live in a `.env` file that is never committed. If those keys are missing, the app still boots and tells you exactly what to set. + +--- + +## What you can do with it + +- Paste a review (or a whole pile of them) and see sentiment, confidence, and key phrases. +- Drop in a PNG or JPEG and get the text back. +- Call the same routes from `curl` or `/docs` if you'd rather skip the UI. +- Run the original batch script over the sample hotel reviews once credentials are in place. + +OCR does not call Azure. It uses the Tesseract binary installed in the API image, so it works with the key fields left blank. + +--- + +## Architecture + +The browser only talks to Streamlit. Streamlit only talks to FastAPI. FastAPI is the only process that sees the Azure key. + +```mermaid +flowchart LR + browser["Browser"] + ui["Streamlit UI
:8501"] + api["FastAPI
:8000"] + azure["Azure AI Language
text analytics v3.1"] + tess["Tesseract OCR
inside the API container"] + + browser --> ui + ui -->|"JSON and image uploads"| api + api -->|"sentiment and key phrases"| azure + api --> tess +``` + +Text analysis, when credentials are present: + +```mermaid +sequenceDiagram + actor You + participant UI as Streamlit + participant API as FastAPI + participant Azure as Azure AI Language + + You->>UI: Paste text and click Analyze + UI->>API: POST /api/v1/analyze + API->>API: Read AZURE_LANGUAGE_ENDPOINT and AZURE_LANGUAGE_KEY + alt credentials missing + API-->>UI: 503 with setup instructions + UI-->>You: Warning, no call to Azure + else credentials present + API->>Azure: POST /text/analytics/v3.1/sentiment + API->>Azure: POST /text/analytics/v3.1/keyPhrases + Azure-->>API: documents + API-->>UI: sentiment, confidence, key phrases + UI-->>You: Result cards + end +``` + +OCR never leaves the machine: + +```mermaid +sequenceDiagram + actor You + participant UI as Streamlit + participant API as FastAPI + participant Tess as Tesseract + + You->>UI: Upload an image + UI->>API: POST /api/v1/extract_text + alt tesseract binary missing + API-->>UI: 503 with install instructions + else tesseract present + API->>Tess: image bytes + Tess-->>API: text + API-->>UI: text plus timing + UI-->>You: Image beside the transcript + end +``` + +--- + +## How the project is put together + +This is the path the code actually takes, from an empty checkout to a result on screen. + +1. **One API, not three.** The repo used to boot two FastAPI apps (`main.py` for Azure text analytics, `fastapi-app/main.py` for OCR) plus a toy async demo (`fastapi-app/main2.py`). Those are now a single app in `app/main.py`. The root `main.py` only re-exports that app so the existing Azure App Service command `gunicorn -k uvicorn.workers.UvicornWorker main:app` still imports the right object. +2. **Configuration is read per request** from the environment in `app/config.py`. Accepted names are `AZURE_LANGUAGE_ENDPOINT` and `AZURE_LANGUAGE_KEY`. The older `COG_SERVICE_ENDPOINT` / `COG_SERVICE_KEY` pair still works. Placeholder values such as `your-key-here` count as unset. +3. **Text analytics** (`app/text_analytics.py`) posts to Azure AI Language REST `v3.1`, in batches of 10 documents. The response keeps the original `sentiment` and `keyphrases` arrays and adds a friendlier `results` list for the UI. The subscription key is sent as a header and is not logged. +4. **OCR** (`app/ocr.py`) opens the upload with Pillow and passes it to Tesseract. Files over 10 MB or files that are not images get a 400. A missing `tesseract` binary gets a 503 that tells you to use the Docker image. +5. **The UI** (`frontend/streamlit_app.py`) is a separate process. `frontend/api_client.py` is the only HTTP client it uses, which is why the same error strings show up in the browser and in `curl`. +6. **The batch script** (`text-analytics-sdk/text-analysis.py`) still walks `text-analytics-sdk/reviews/*.txt` and uses the Azure SDK. It shares `app/config.py`, so a missing key prints the same instructions and exits 1 instead of throwing a stack trace. +7. **Docker** builds two images from `Dockerfile` and `Dockerfile.frontend`. `docker-compose.yml` wires them together and publishes ports 8000 and 8501. + +--- + +## Run it with Docker + +You need Docker with Compose v2. + +```bash +cp .env.example .env +``` + +Open `.env` and, when you have an Azure AI Language resource, set: + +```bash +AZURE_LANGUAGE_ENDPOINT=https://.cognitiveservices.azure.com +AZURE_LANGUAGE_KEY= +``` + +Blank values are valid. You can `docker compose up` before you create the Azure resource. + +```bash +docker compose up --build +``` + +Then open: + +| Surface | URL | +| --- | --- | +| Workbench UI | http://localhost:8501 | +| API docs | http://localhost:8000/docs | +| Health | http://localhost:8000/health | + +## Demo + +Leave the Azure fields in `.env` empty and walk the running stack once. This is the same path as a local screen recording of the Compose services: the UI, OCR on the sample image, the missing-key message, then the API docs. + +1. Open http://localhost:8501. The page title is **Text & OCR Workbench**. One card says Azure Language credentials are not configured and names `AZURE_LANGUAGE_ENDPOINT` and `AZURE_LANGUAGE_KEY`. The other says Tesseract OCR is available. +2. Open the **Read an image** tab. Choose `samples/ocr-sample.png` and click **Extract text**. The picture sits beside the transcript. On this sample that transcript includes "The quick brown dog jumped over the lazy fox." +3. Open the **Analyze text** tab. Click **Load example review**, then **Analyze text**. A second card under the button repeats the configuration message. The API returns HTTP 503 and does not call Azure. +4. Open http://localhost:8000/docs. The Swagger page lists `/health`, `/api/v1/status`, `POST /api/v1/analyze`, and `POST /api/v1/extract_text`. + +When you later paste a real endpoint and key into `.env` and run `docker compose up --build` again, step 3 returns sentiment and key phrases instead of that message. + +Stop it with Ctrl+C, or `docker compose down` from another terminal. + +The first build installs Python dependencies and the `tesseract-ocr` package. Later starts are just container boots. + +### Try the API without the UI + +No credentials: + +```bash +curl -s -X POST http://localhost:8000/api/v1/analyze \ + -H 'content-type: application/json' \ + -d '{"text_to_analyze":["The rooms were spotless."]}' +``` + +You should get HTTP 503 and a message that names `AZURE_LANGUAGE_ENDPOINT` and `AZURE_LANGUAGE_KEY`. + +OCR, using the sample image checked into the repo: + +```bash +curl -s -F "images=@samples/ocr-sample.png" \ + http://localhost:8000/api/v1/extract_text +``` + +The body includes `results` (a list of filename and text) and the original filename-to-text map, plus `time_taken_seconds`. + +With real credentials in `.env`, the same analyze `curl` returns sentiment and key phrases. Restart the stack after editing `.env` so Compose picks up the new values: `docker compose up --build`. --- -## Building APIs with Python using the FastAPI and Microsoft Azure Cognitive Services +## Run it without Docker + +Use Python 3.12. OCR on the host also needs the Tesseract binary (`sudo apt-get install tesseract-ocr` on Debian or Ubuntu). -This project is a simple demonstration of how to deploy a FastAPI model on Microsoft Azure, complete with CI/CD functionality. This repo also contains 2 mini applications. One of them runs FastAPI locally with PyTesseract to perform Optical Character Recognition, and the other uses the Azure CognitiveServices Python SDK to perform Text Analytics, using key and endpoint acquired from the Azure Cognitive Services Resource. +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements-dev.txt +cp .env.example .env +# edit .env if you have Azure keys, then: +set -a && source .env && set +a + +uvicorn app.main:app --reload --port 8000 +``` + +In a second terminal, with the same virtualenv: + +```bash +streamlit run frontend/streamlit_app.py +``` + +Streamlit defaults to `http://localhost:8000` for the API. Override it with `API_BASE_URL` if the API is somewhere else. --- -
- -
+## Using the UI -### Deploying REST API on Microsoft Azure +1. The top of the page reports whether Azure Language is configured and whether Tesseract is installed. A warning there is the app working as designed, not a crash. +2. **Analyze text.** Paste copy, or click **Load example review**, pick a language, and click **Analyze text**. Each document comes back as a card: sentiment chip, confidence, key phrases. The raw JSON sits in an expander. +3. **Read an image.** Upload one or more PNG, JPEG, TIFF, BMP, WEBP, or GIF files (10 MB each) and click **Extract text**. The picture stays next to the transcript. -After pushing the code to GitHub, the following steps are followed to get the model deployed on Azure +Colors are a light page with dark text. Sentiment chips use dark lettering on a tinted background so they stay readable. --- -#### Create an App Service Resource and connect to GitHub for Continuous Integration/ Continuous Deployment +## Batch reviews -A WSGI server is needed to deploy a FastAPI project on Azure App Service. I used `gunicorn` in this project. The startup script can thus have the gunicorn command to spin up the FastAPI app with the help of Gunicorn's worker class `uvicorn.workers.UvicornWorker`. The startup command is given as: +Five sample reviews live in `text-analytics-sdk/reviews/`. The SDK script scores all of them in one go: +```bash +set -a && source .env && set +a +python text-analytics-sdk/text-analysis.py ``` -gunicorn -w 2 -k uvicorn.workers.UvicornWorker main:app + +Without keys, it prints the configuration message and exits 1. It does not open a connection to Azure. + +--- + +## Tests + +```bash +pip install -r requirements-dev.txt +pytest ``` +The suite covers: + +- health, status, and the legacy `POST /` route +- HTTP 503 with the setup message when Azure keys are missing or still placeholders +- a mocked Azure success path, including batches larger than 10 documents +- HTTP 502 when Azure rejects the key, without echoing the key +- OCR with Tesseract mocked, a non-image upload, an oversized upload, and the sample PNG when the `tesseract` binary is installed +- the UI client's error formatting +- the batch script's missing-credential exit + +GitHub Actions (`.github/workflows/test.yml`) runs that suite and also builds the Compose stack, then checks health, the 503 analyze response, OCR on the sample image, and the Streamlit health endpoint. + +`.github/workflows/main_text-analytics-fastapi.yml` still deploys **only the API** to the existing Azure App Service named `text-analytics-fastapi` on pushes to `main`. That service needs `AZURE_LANGUAGE_ENDPOINT` and `AZURE_LANGUAGE_KEY` in its application settings or text analysis will return 503. The UI is not part of that deploy. Replacing this workflow is the first hosting task in [PRODUCTIONIZING.md](PRODUCTIONIZING.md). + --- -#### Check the GitHub Actions and Test the App Service +## Routes + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/health` | Liveness. Stays green when Azure is not configured. | +| `GET` | `/api/v1/status` | Whether Azure and Tesseract are ready. No secrets. | +| `POST` | `/api/v1/analyze` | Body: `{"text_to_analyze": ["..."], "language": "en"}`. Up to 25 documents, 5120 characters each. | +| `POST` | `/` | Same handler as `/api/v1/analyze`, for the original App Service clients. | +| `POST` | `/api/v1/extract_text` | Multipart field `images`, one or more files. | + +`language` is a short code such as `en` or `zh-Hans`. -A workflow file, `/.github/workflows/main_text_analytics-fastapi.yml`, is added automatically by Azure, and used to configure the CI/CD with GitHub for Authentication, Build and Deployment jobs required for this model pipeline. Once the deployment is successful, the API is ready to be used from the url +--- + +## Project layout + +```text +app/ FastAPI package (config, text analytics, OCR, routes) +frontend/ Streamlit UI and its API client +text-analytics-sdk/ Batch SDK script and sample reviews +samples/ ocr-sample.png used by tests and the curl example +tests/ pytest suite +Dockerfile API image (Python 3.12 + tesseract-ocr) +Dockerfile.frontend Streamlit image +docker-compose.yml api + ui +.env.example committed template, no real keys +PRODUCTIONIZING.md what to do before this faces the public internet +``` --- -#### Create An Application Insights Resource on Azure and Implement logging in the API source code +## Keys, logs, and old entry points -This resource helps us log data from the API. An additional python package, `opencensus-ext-azure` is required to implement logging in the API. Once the code is commited and pushed to GitHub, the API is automatically redeployed, and if successful, then the logs can be viewed on Azure. +- Real credentials belong in `.env` only. `.gitignore` already ignores that file. `.env.example` ships with empty values. +- An Azure subscription key and an Application Insights instrumentation key used to be hardcoded in `main.py`. They are gone from the tree. Treat them as compromised and rotate them. [PRODUCTIONIZING.md](PRODUCTIONIZING.md) has the steps. Do not put the old values back. +- Logs record status codes, document counts, and Azure HTTP failures. They do not record the subscription key or the document text. +- Removed as duplicate entry points: `fastapi-app/main.py`, `fastapi-app/main2.py`, `fastapi-app/ocr.py`, `fastapi-app/utils.py`, and the root `utils.py`. The sample OCR image moved to `samples/ocr-sample.png`. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..37e81d9 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""HTTP API for Azure AI Language text analytics and local OCR.""" diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..0687c4c --- /dev/null +++ b/app/config.py @@ -0,0 +1,94 @@ +"""Runtime configuration loaded from the environment. + +The API starts even when Azure credentials are absent. Text analysis then +returns :data:`MISSING_CREDENTIALS_MESSAGE` instead of crashing or calling +Azure with an empty key. Image OCR does not read these settings. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +# Obvious stand-ins from .env.example. Treating them as "unset" means a copied +# example file still gets the configuration message rather than a 401 from Azure. +_PLACEHOLDER_KEYS = { + "your-key-here", + "changeme", + "", + "paste-your-key-here", +} +_PLACEHOLDER_ENDPOINT_MARKERS = ( + "your-resource", + "your-language-resource", + "example.cognitiveservices.azure.com", +) + +MISSING_CREDENTIALS_MESSAGE = ( + "Azure Language credentials are not configured. " + "Copy .env.example to .env, set AZURE_LANGUAGE_ENDPOINT and " + "AZURE_LANGUAGE_KEY to the endpoint and key of an Azure AI Language " + "resource, then restart the API. " + "Sentiment and key-phrase analysis stay unavailable until those values " + "are real. Image OCR does not use them." +) + +OCR_UNAVAILABLE_MESSAGE = ( + "Tesseract OCR is not installed. " + "Run the API with Docker Compose (the API image installs tesseract-ocr), " + "or install the tesseract binary on the host and restart the API." +) + + +@dataclass(frozen=True) +class Settings: + """Azure Language connection details, when both values are present.""" + + azure_language_endpoint: str | None + azure_language_key: str | None + + @property + def azure_language_configured(self) -> bool: + """True when both the endpoint and the key look usable.""" + return bool(self.azure_language_endpoint and self.azure_language_key) + + +def _clean(value: str | None) -> str: + return (value or "").strip() + + +def _usable_endpoint(endpoint: str) -> str: + """Drop a trailing slash and reject example hostnames.""" + cleaned = endpoint.rstrip("/") + lowered = cleaned.lower() + if not cleaned: + return "" + if any(marker in lowered for marker in _PLACEHOLDER_ENDPOINT_MARKERS): + return "" + return cleaned + + +def _usable_key(key: str) -> str: + if key.lower() in _PLACEHOLDER_KEYS: + return "" + return key + + +def load_settings() -> Settings: + """Read Azure Language settings from the process environment. + + ``AZURE_LANGUAGE_ENDPOINT`` and ``AZURE_LANGUAGE_KEY`` are the names this + project documents. The original batch script's ``COG_SERVICE_ENDPOINT`` + and ``COG_SERVICE_KEY`` are still accepted so an existing local ``.env`` + keeps working. + """ + endpoint = _usable_endpoint( + _clean(os.getenv("AZURE_LANGUAGE_ENDPOINT") or os.getenv("COG_SERVICE_ENDPOINT")) + ) + key = _usable_key( + _clean(os.getenv("AZURE_LANGUAGE_KEY") or os.getenv("COG_SERVICE_KEY")) + ) + return Settings( + azure_language_endpoint=endpoint or None, + azure_language_key=key or None, + ) diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 0000000..05feea5 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,25 @@ +"""Errors the API turns into clear HTTP responses. + +These are raised by the text-analytics and OCR services and handled in +``app.main`` so callers see a message they can act on. +""" + + +class CredentialsNotConfigured(Exception): + """Azure Language settings are missing or still placeholders.""" + + +class AzureLanguageError(Exception): + """Azure AI Language rejected the call or could not be reached.""" + + def __init__(self, message: str, status_code: int = 502) -> None: + super().__init__(message) + self.status_code = status_code + + +class OcrUnavailable(Exception): + """The Tesseract binary is not installed on this machine.""" + + +class InvalidImage(Exception): + """An upload is empty, too large, or not a readable image.""" diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..9e9c941 --- /dev/null +++ b/app/main.py @@ -0,0 +1,209 @@ +"""FastAPI application: text analytics and OCR. + +Run it with ``uvicorn app.main:app`` or, for the historical App Service +command, ``gunicorn -k uvicorn.workers.UvicornWorker main:app``. The root +``main.py`` module re-exports this ``app`` object. +""" + +from __future__ import annotations + +import logging +import re +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from fastapi import FastAPI, File, HTTPException, UploadFile +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field, field_validator + +from app.config import ( + MISSING_CREDENTIALS_MESSAGE, + OCR_UNAVAILABLE_MESSAGE, + load_settings, +) +from app.errors import ( + AzureLanguageError, + CredentialsNotConfigured, + InvalidImage, + OcrUnavailable, +) +from app.ocr import extract_text, tesseract_available +from app.text_analytics import analyze_texts + +logger = logging.getLogger(__name__) + +_LANGUAGE_CODE = re.compile(r"^[a-zA-Z]{2,3}(-[a-zA-Z0-9]{2,8})?$") +_MAX_DOCUMENTS = 25 +_MAX_DOCUMENT_CHARS = 5120 + + +class AnalyzeRequest(BaseModel): + """Documents to send to Azure AI Language.""" + + text_to_analyze: list[str] = Field(..., min_length=1, max_length=_MAX_DOCUMENTS) + language: str = "en" + + @field_validator("text_to_analyze") + @classmethod + def texts_must_be_usable(cls, texts: list[str]) -> list[str]: + cleaned: list[str] = [] + for text in texts: + stripped = text.strip() + if not stripped: + raise ValueError("Each entry in text_to_analyze must contain text.") + if len(stripped) > _MAX_DOCUMENT_CHARS: + raise ValueError( + f"Each document must be {_MAX_DOCUMENT_CHARS} characters or fewer." + ) + cleaned.append(stripped) + return cleaned + + @field_validator("language") + @classmethod + def language_must_be_a_code(cls, language: str) -> str: + candidate = language.strip() + if not _LANGUAGE_CODE.fullmatch(candidate): + raise ValueError("language must be a short code such as en or en-US.") + return candidate + + +def _status_payload() -> dict[str, Any]: + settings = load_settings() + ocr_ready = tesseract_available() + messages: list[str] = [] + if settings.azure_language_configured: + messages.append("Azure Language credentials are configured.") + else: + messages.append(MISSING_CREDENTIALS_MESSAGE) + if ocr_ready: + messages.append("Tesseract OCR is available.") + else: + messages.append(OCR_UNAVAILABLE_MESSAGE) + return { + "azure_language_configured": settings.azure_language_configured, + "ocr_available": ocr_ready, + "messages": messages, + } + + +@asynccontextmanager +async def lifespan(_app: FastAPI) -> AsyncIterator[None]: + """Log which backends are ready. Never log secret values.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s") + for message in _status_payload()["messages"]: + if message in (MISSING_CREDENTIALS_MESSAGE, OCR_UNAVAILABLE_MESSAGE): + logger.warning(message) + else: + logger.info(message) + yield + + +app = FastAPI( + title="Text & OCR Workbench API", + description=( + "Sentiment and key phrases through Azure AI Language, and image OCR " + "through Tesseract. Azure credentials come from the environment. " + "When they are missing, text analysis returns HTTP 503 with setup instructions." + ), + version="1.1.0", + lifespan=lifespan, +) + + +@app.exception_handler(CredentialsNotConfigured) +async def credentials_not_configured(_request: object, _exc: CredentialsNotConfigured) -> JSONResponse: + return JSONResponse(status_code=503, content={"detail": MISSING_CREDENTIALS_MESSAGE}) + + +@app.exception_handler(OcrUnavailable) +async def ocr_unavailable(_request: object, _exc: OcrUnavailable) -> JSONResponse: + return JSONResponse(status_code=503, content={"detail": OCR_UNAVAILABLE_MESSAGE}) + + +@app.exception_handler(AzureLanguageError) +async def azure_language_error(_request: object, exc: AzureLanguageError) -> JSONResponse: + return JSONResponse(status_code=exc.status_code, content={"detail": str(exc)}) + + +@app.exception_handler(InvalidImage) +async def invalid_image(_request: object, exc: InvalidImage) -> JSONResponse: + return JSONResponse(status_code=400, content={"detail": str(exc)}) + + +@app.get("/health") +def health() -> dict[str, str]: + """Liveness probe. This stays healthy when Azure is not configured.""" + return {"status": "ok"} + + +@app.get("/api/v1/status") +def status() -> dict[str, Any]: + """Report whether text analysis and OCR can run. Does not reveal secrets.""" + return _status_payload() + + +@app.get("/") +def home() -> dict[str, str]: + """Point a browser at the interactive docs and the Streamlit UI.""" + return { + "message": ( + "Text & OCR API. Interactive docs are at /docs. " + "The Streamlit UI listens on port 8501 when you use Docker Compose." + ), + "analyze": "POST /api/v1/analyze", + "ocr": "POST /api/v1/extract_text", + "health": "GET /health", + "status": "GET /api/v1/status", + } + + +def _analyze(body: AnalyzeRequest) -> dict[str, Any]: + return analyze_texts(body.text_to_analyze, language=body.language) + + +@app.post("/api/v1/analyze") +def analyze(body: AnalyzeRequest) -> dict[str, Any]: + """Sentiment and key phrases for one or more documents.""" + return _analyze(body) + + +@app.post("/") +def analyze_legacy(body: AnalyzeRequest) -> dict[str, Any]: + """Same handler as ``/api/v1/analyze``. + + The original App Service exposed text analysis on ``POST /``. + """ + return _analyze(body) + + +@app.post("/api/v1/extract_text") +async def extract_text_route(images: list[UploadFile] = File(...)) -> dict[str, Any]: + """OCR each uploaded image. The response keeps the original filename map.""" + if not images: + raise HTTPException(status_code=400, detail="Upload at least one image.") + + started = time.perf_counter() + results: list[dict[str, str]] = [] + by_filename: dict[str, str] = {} + for upload in images: + filename = upload.filename or "upload" + contents = await upload.read() + try: + text = extract_text(contents) + except InvalidImage as exc: + # Include the filename so a multi-file request says which one failed. + raise InvalidImage(f"{filename}: {exc}") from exc + results.append({"filename": filename, "text": text}) + by_filename[filename] = text + + elapsed = round(time.perf_counter() - started, 2) + response: dict[str, Any] = { + "results": results, + "time_taken_seconds": elapsed, + # Original OCR route used this key. Kept so older clients still parse. + "Time Taken": elapsed, + } + response.update(by_filename) + return response diff --git a/app/ocr.py b/app/ocr.py new file mode 100644 index 0000000..d49da67 --- /dev/null +++ b/app/ocr.py @@ -0,0 +1,58 @@ +"""Local OCR through Tesseract. + +The binary is provided by the API Docker image (``tesseract-ocr``). A host +install works the same way. When the binary is missing, callers get +:class:`app.errors.OcrUnavailable` and a message that says how to fix it. +""" + +from __future__ import annotations + +import io +import logging +import shutil + +import pytesseract +from PIL import Image, UnidentifiedImageError + +from app.errors import InvalidImage, OcrUnavailable + +logger = logging.getLogger(__name__) + +MAX_IMAGE_BYTES = 10 * 1024 * 1024 + + +def tesseract_available() -> bool: + """True when the ``tesseract`` executable is on ``PATH``.""" + return shutil.which("tesseract") is not None + + +def extract_text(image_bytes: bytes, lang: str = "eng") -> str: + """Read text from image bytes. + + Raises: + OcrUnavailable: Tesseract is not installed. + InvalidImage: the bytes are empty, too large, or not an image. + """ + if not tesseract_available(): + raise OcrUnavailable() + if not image_bytes: + raise InvalidImage("The uploaded file is empty.") + if len(image_bytes) > MAX_IMAGE_BYTES: + raise InvalidImage("Each image must be 10 MB or smaller.") + + try: + with Image.open(io.BytesIO(image_bytes)) as image: + # Load now so a truncated file fails here, before Tesseract. + image.load() + text = pytesseract.image_to_string(image, lang=lang) + except UnidentifiedImageError as exc: + raise InvalidImage("The uploaded file is not a readable image.") from exc + except OSError as exc: + raise InvalidImage("The uploaded file could not be opened as an image.") from exc + except pytesseract.TesseractNotFoundError as exc: + raise OcrUnavailable() from exc + except pytesseract.TesseractError as exc: + logger.warning("Tesseract failed: %s", exc) + raise InvalidImage("Tesseract could not read that image.") from exc + + return text.strip() diff --git a/app/text_analytics.py b/app/text_analytics.py new file mode 100644 index 0000000..39da56d --- /dev/null +++ b/app/text_analytics.py @@ -0,0 +1,191 @@ +"""Sentiment and key-phrase calls against Azure AI Language. + +The REST shape matches the original root API: one request per operation to +``/text/analytics/v3.1/{sentiment|keyPhrases}``. Documents are sent in batches +of ten, which is the service limit for these routes. +""" + +from __future__ import annotations + +import logging +from typing import Any + +import requests + +from app.config import load_settings +from app.errors import AzureLanguageError, CredentialsNotConfigured + +logger = logging.getLogger(__name__) + +_BATCH_SIZE = 10 +_TIMEOUT_SECONDS = 30 + + +def _analytics_url(endpoint: str, route: str) -> str: + """Build a v3.1 URL, even if the env value already includes the prefix.""" + base = endpoint.rstrip("/") + marker = "/text/analytics/v3.1" + if base.lower().endswith(marker): + return f"{base}/{route}" + return f"{base}{marker}/{route}" + + +def _headers(key: str) -> dict[str, str]: + return { + "Ocp-Apim-Subscription-Key": key, + "Content-Type": "application/json", + "Accept": "application/json", + } + + +def _error_message(payload: Any) -> str: + """Pull a human-readable message out of an Azure error body.""" + if not isinstance(payload, dict): + return "" + error = payload.get("error") + if isinstance(error, dict): + message = error.get("message") or error.get("innererror", {}) + if isinstance(message, str) and message.strip(): + return message.strip() + if isinstance(message, dict): + inner = message.get("message") + if isinstance(inner, str) and inner.strip(): + return inner.strip() + message = payload.get("message") + if isinstance(message, str): + return message.strip() + return "" + + +def _post(endpoint: str, key: str, route: str, documents: list[dict[str, Any]]) -> dict[str, Any]: + """POST one batch. The subscription key is sent as a header and never logged.""" + url = _analytics_url(endpoint, route) + try: + response = requests.post( + url, + headers=_headers(key), + json={"documents": documents}, + timeout=_TIMEOUT_SECONDS, + ) + except requests.RequestException as exc: + logger.warning("Azure Language request to %s failed: %s", route, exc.__class__.__name__) + raise AzureLanguageError( + "Could not reach Azure AI Language. " + "Check AZURE_LANGUAGE_ENDPOINT and that this machine can reach Azure." + ) from exc + + try: + payload: Any = response.json() + except ValueError: + payload = {} + + if response.status_code in (401, 403): + logger.info("Azure Language rejected credentials for %s (HTTP %s)", route, response.status_code) + raise AzureLanguageError( + "Azure rejected the Language credentials " + f"(HTTP {response.status_code}). " + "Check AZURE_LANGUAGE_KEY and that AZURE_LANGUAGE_ENDPOINT belongs to " + "that same Language resource." + ) + + if response.status_code >= 400: + message = _error_message(payload) or ( + f"Azure AI Language returned HTTP {response.status_code} for {route}." + ) + logger.info("Azure Language %s failed: HTTP %s", route, response.status_code) + raise AzureLanguageError(message) + + if not isinstance(payload, dict): + raise AzureLanguageError(f"Azure AI Language returned an unexpected payload for {route}.") + return payload + + +def _index_documents(payload: dict[str, Any]) -> dict[str, dict[str, Any]]: + documents = payload.get("documents") or [] + indexed: dict[str, dict[str, Any]] = {} + if isinstance(documents, list): + for document in documents: + if isinstance(document, dict) and "id" in document: + indexed[str(document["id"])] = document + return indexed + + +def _index_errors(payload: dict[str, Any]) -> dict[str, str]: + errors = payload.get("errors") or [] + indexed: dict[str, str] = {} + if not isinstance(errors, list): + return indexed + for item in errors: + if not isinstance(item, dict) or "id" not in item: + continue + error = item.get("error") + message = "" + if isinstance(error, dict): + message = str(error.get("message") or "").strip() + indexed[str(item["id"])] = message or "Azure could not analyze this document." + return indexed + + +def analyze_texts(texts: list[str], language: str = "en") -> dict[str, Any]: + """Run sentiment and key-phrase extraction for each text. + + Raises: + CredentialsNotConfigured: endpoint or key is missing. + AzureLanguageError: Azure returned an error or could not be reached. + """ + settings = load_settings() + if not settings.azure_language_configured: + raise CredentialsNotConfigured() + + endpoint = settings.azure_language_endpoint + key = settings.azure_language_key + # The check above guarantees both are set. The locals keep the type narrow. + assert endpoint is not None and key is not None + + sentiment_documents: list[dict[str, Any]] = [] + keyphrase_documents: list[dict[str, Any]] = [] + results: list[dict[str, Any]] = [] + + for start in range(0, len(texts), _BATCH_SIZE): + chunk = texts[start : start + _BATCH_SIZE] + documents = [ + {"id": str(start + index + 1), "language": language, "text": text} + for index, text in enumerate(chunk) + ] + sentiment_payload = _post(endpoint, key, "sentiment", documents) + keyphrase_payload = _post(endpoint, key, "keyPhrases", documents) + sentiment_by_id = _index_documents(sentiment_payload) + phrases_by_id = _index_documents(keyphrase_payload) + errors = _index_errors(sentiment_payload) + errors.update(_index_errors(keyphrase_payload)) + + for document in documents: + doc_id = str(document["id"]) + sentiment_doc = sentiment_by_id.get(doc_id) + phrase_doc = phrases_by_id.get(doc_id) + if sentiment_doc is not None: + sentiment_documents.append(sentiment_doc) + if phrase_doc is not None: + keyphrase_documents.append(phrase_doc) + + item: dict[str, Any] = { + "id": doc_id, + "text": document["text"], + "sentiment": None if sentiment_doc is None else sentiment_doc.get("sentiment"), + "confidence_scores": None + if sentiment_doc is None + else sentiment_doc.get("confidenceScores"), + "key_phrases": [] if phrase_doc is None else list(phrase_doc.get("keyPhrases") or []), + } + if doc_id in errors and sentiment_doc is None: + item["error"] = errors[doc_id] + elif doc_id in errors: + item["warning"] = errors[doc_id] + results.append(item) + + logger.info("Analyzed %s document(s) with Azure Language", len(texts)) + return { + "sentiment": sentiment_documents, + "keyphrases": keyphrase_documents, + "results": results, + } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ccce4e8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,46 @@ +# Local stack: API on http://localhost:8000 and the UI on http://localhost:8501. +# +# cp .env.example .env # optional; empty values are fine +# docker compose up --build +# +# Azure keys are read from .env when that file exists. They are never baked +# into the images. With no .env, or with blank keys, the API still starts and +# text analysis answers with a configuration message. + +services: + api: + build: + context: . + dockerfile: Dockerfile + ports: + - "8000:8000" + # Compose reads a project-root .env for ${VAR} substitution. Missing file + # or blank values become empty strings, and the API explains that text + # analysis needs real credentials. The key is not copied into the image. + environment: + AZURE_LANGUAGE_ENDPOINT: ${AZURE_LANGUAGE_ENDPOINT:-} + AZURE_LANGUAGE_KEY: ${AZURE_LANGUAGE_KEY:-} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health')"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 15s + + ui: + build: + context: . + dockerfile: Dockerfile.frontend + ports: + - "8501:8501" + environment: + API_BASE_URL: http://api:8000 + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8501/_stcore/health')"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 20s diff --git a/fastapi-app/main.py b/fastapi-app/main.py deleted file mode 100644 index ca5785c..0000000 --- a/fastapi-app/main.py +++ /dev/null @@ -1,30 +0,0 @@ -from fastapi import FastAPI, File, UploadFile -from typing import List -import time -import asyncio -import ocr -import utils - -app = FastAPI() - - -@app.get("/") -def home(): - return {"message": "Visit the endpoint: /api/v1/extract_text to perform OCR."} - - -@app.post("/api/v1/extract_text") -async def extract_text(Images: List[UploadFile] = File(...)): - response = {} - s = time.time() - tasks = [] - for img in Images: - print("Images Uploaded: ", img.filename) - temp_file = utils._save_file_to_server( - img, path="./", save_as=img.filename) - tasks.append(asyncio.create_task(ocr.read_image(temp_file))) - text = await asyncio.gather(*tasks) - for i in range(len(text)): - response[Images[i].filename] = text[i] - response["Time Taken"] = round((time.time() - s), 2) - return response diff --git a/fastapi-app/main2.py b/fastapi-app/main2.py deleted file mode 100644 index 8e88a5a..0000000 --- a/fastapi-app/main2.py +++ /dev/null @@ -1,27 +0,0 @@ -from fastapi import FastAPI -import time -import asyncio - -app = FastAPI() - - -@app.get("/") -async def home(): - tasks = [] - start = time.time() - for i in range(2): - tasks.append(asyncio.create_task(func1())) - tasks.append(asyncio.create_task(func2())) - response = await asyncio.gather(*tasks) - end = time.time() - return {"response": response, "time_taken": (end - start)} - - -async def func1(): - await asyncio.sleep(2) - return "Func1() Completed" - - -async def func2(): - await asyncio.sleep(1) - return "Func2() Completed" diff --git a/fastapi-app/ocr.py b/fastapi-app/ocr.py deleted file mode 100644 index caecf65..0000000 --- a/fastapi-app/ocr.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytesseract -import asyncio - -# pytesseract.pytesseract.tesseract_cmd = "C:\\Program Files\\Tesseract-OCR\\tesseract.exe" - - -async def read_image(img_path, lang='eng'): - try: - text = pytesseract.image_to_string(img_path, lang=lang) - await asyncio.sleep(2) - return text - except: - return "[ERROR] Unable to process file: {0}".format(img_path) diff --git a/fastapi-app/test.png.png b/fastapi-app/test.png.png deleted file mode 100644 index 1b699c9..0000000 Binary files a/fastapi-app/test.png.png and /dev/null differ diff --git a/fastapi-app/utils.py b/fastapi-app/utils.py deleted file mode 100644 index 1e9620b..0000000 --- a/fastapi-app/utils.py +++ /dev/null @@ -1,12 +0,0 @@ -import shutil -import os - - -def _save_file_to_server(uploaded_file, path=".", save_as="default"): - extension = os.path.splitext(uploaded_file.filename)[-1] - temp_file = os.path.join(path, save_as + extension) - - with open(temp_file, "wb") as buffer: - shutil.copyfileobj(uploaded_file.file, buffer) - - return temp_file diff --git a/frontend/__init__.py b/frontend/__init__.py new file mode 100644 index 0000000..e554e19 --- /dev/null +++ b/frontend/__init__.py @@ -0,0 +1 @@ +"""Streamlit UI and the small HTTP client it uses to talk to the API.""" diff --git a/frontend/api_client.py b/frontend/api_client.py new file mode 100644 index 0000000..c010a68 --- /dev/null +++ b/frontend/api_client.py @@ -0,0 +1,126 @@ +"""HTTP client used by the Streamlit UI. + +The UI never talks to Azure itself. It calls the FastAPI service so both +surfaces share the same credential checks and error messages. +""" + +from __future__ import annotations + +import os +from typing import Any + +import requests + +DEFAULT_API_BASE_URL = "http://localhost:8000" +_TIMEOUT_SECONDS = 120 + + +class ApiConnectionError(Exception): + """The API process is not reachable at ``API_BASE_URL``.""" + + +class ApiError(Exception): + """The API responded with an error payload.""" + + def __init__(self, status_code: int, detail: str) -> None: + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +def api_base_url() -> str: + """Base URL of the FastAPI service, without a trailing slash.""" + return os.getenv("API_BASE_URL", DEFAULT_API_BASE_URL).strip().rstrip("/") or DEFAULT_API_BASE_URL + + +def format_detail(payload: Any, fallback: str) -> str: + """Turn FastAPI's ``detail`` field into one readable sentence. + + Validation errors arrive as a list of objects. Configuration and Azure + errors arrive as a string. + """ + if isinstance(payload, dict) and "detail" in payload: + detail = payload["detail"] + else: + detail = payload + + if isinstance(detail, str) and detail.strip(): + return detail.strip() + if isinstance(detail, list): + parts: list[str] = [] + for item in detail: + if isinstance(item, dict): + message = str(item.get("msg") or item.get("message") or "").strip() + loc = item.get("loc") or [] + field = ".".join(str(part) for part in loc if part not in ("body",)) + if message and field: + parts.append(f"{field}: {message}") + elif message: + parts.append(message) + elif isinstance(item, str) and item.strip(): + parts.append(item.strip()) + if parts: + return " ".join(parts) + return fallback + + +def _request(method: str, path: str, **kwargs: Any) -> Any: + url = f"{api_base_url()}{path}" + try: + response = requests.request(method, url, timeout=_TIMEOUT_SECONDS, **kwargs) + except requests.RequestException as exc: + raise ApiConnectionError( + f"Could not reach the API at {api_base_url()}. " + "Start it with Docker Compose (`docker compose up --build`) " + "or run `uvicorn app.main:app --port 8000`." + ) from exc + + if response.status_code >= 400: + try: + payload: Any = response.json() + except ValueError: + payload = {} + raise ApiError( + response.status_code, + format_detail(payload, f"The API returned HTTP {response.status_code}."), + ) + + try: + return response.json() + except ValueError as exc: + raise ApiError(response.status_code, "The API returned a response that was not JSON.") from exc + + +def get_status() -> dict[str, Any]: + """Return the API's readiness report.""" + payload = _request("GET", "/api/v1/status") + if not isinstance(payload, dict): + raise ApiError(200, "The API status response had an unexpected shape.") + return payload + + +def analyze_text(texts: list[str], language: str = "en") -> dict[str, Any]: + """Ask the API to score sentiment and extract key phrases.""" + payload = _request( + "POST", + "/api/v1/analyze", + json={"text_to_analyze": texts, "language": language}, + ) + if not isinstance(payload, dict): + raise ApiError(200, "The API analysis response had an unexpected shape.") + return payload + + +def extract_text(files: list[tuple[str, bytes, str]]) -> dict[str, Any]: + """Send images to the OCR route. + + Each tuple is ``(filename, content, content_type)``. + """ + multipart = [ + ("images", (filename, content, content_type)) + for filename, content, content_type in files + ] + payload = _request("POST", "/api/v1/extract_text", files=multipart) + if not isinstance(payload, dict): + raise ApiError(200, "The API OCR response had an unexpected shape.") + return payload diff --git a/frontend/streamlit_app.py b/frontend/streamlit_app.py new file mode 100644 index 0000000..dfc10ca --- /dev/null +++ b/frontend/streamlit_app.py @@ -0,0 +1,268 @@ +"""Browser UI for the text-analytics and OCR API. + +Run from the repository root: + + streamlit run frontend/streamlit_app.py + +Docker Compose sets ``API_BASE_URL`` to the API service. On a laptop the +client defaults to ``http://localhost:8000``. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +# Streamlit puts the script directory on sys.path, not the repository root. +# Adding the root lets `frontend` import the same way the tests do. +_ROOT = Path(__file__).resolve().parents[1] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + +import streamlit as st + +from frontend.api_client import ApiConnectionError, ApiError, analyze_text, extract_text, get_status + +# Dark text on tinted backgrounds. Streamlit's light theme supplies the page +# colors; these chips stay readable if a custom theme shifts the card color. +_SENTIMENT_COLORS = { + "positive": ("#14532d", "#dcfce7"), + "negative": ("#7f1d1d", "#fee2e2"), + "neutral": ("#1f2937", "#e5e7eb"), + "mixed": ("#78350f", "#fef3c7"), +} + +_EXAMPLE_TEXT = ( + "The Royal Hotel in London was a pleasure. Clean rooms, a peaceful courtyard, " + "and a short walk to Westminster Abbey. The restaurant next door served a " + "fabulous taster menu. Thoroughly recommended." +) + + +def _sentiment_chip(label: str | None) -> str: + key = (label or "unknown").lower() + foreground, background = _SENTIMENT_COLORS.get(key, ("#1c1917", "#f5f5f4")) + shown = label or "unknown" + return ( + f"{shown}" + ) + + +def _render_status() -> None: + """Show whether Azure and Tesseract are ready. Stop if the API is down.""" + try: + status = get_status() + except ApiConnectionError as exc: + st.error(str(exc)) + st.stop() + except ApiError as exc: + st.error(exc.detail) + st.stop() + + messages = status.get("messages") or [] + azure_ready = bool(status.get("azure_language_configured")) + ocr_ready = bool(status.get("ocr_available")) + + left, right = st.columns(2) + with left: + if azure_ready: + st.success("Azure Language is configured.") + else: + st.warning(next((msg for msg in messages if "not configured" in msg), messages[0] if messages else "")) + with right: + if ocr_ready: + st.success("Tesseract OCR is available.") + else: + st.warning(next((msg for msg in messages if "Tesseract" in msg), "Tesseract OCR is not available.")) + + +def _render_analysis(payload: dict[str, Any]) -> None: + results = payload.get("results") or [] + if not results: + st.info("Azure returned no per-document results.") + return + + for item in results: + title = item.get("sentiment") or "result" + with st.container(border=True): + st.markdown(_sentiment_chip(item.get("sentiment")), unsafe_allow_html=True) + st.caption(f"Document {item.get('id', '')}") + st.write(item.get("text") or "") + if item.get("error"): + st.error(item["error"]) + continue + if item.get("warning"): + st.warning(item["warning"]) + + scores = item.get("confidence_scores") or {} + if isinstance(scores, dict) and scores: + score_cols = st.columns(3) + for column, label in zip(score_cols, ("positive", "neutral", "negative")): + value = float(scores.get(label) or 0) + column.metric(label.capitalize(), f"{value:.0%}") + column.progress(min(max(value, 0.0), 1.0)) + + phrases = item.get("key_phrases") or [] + if phrases: + st.markdown("**Key phrases**") + st.write(", ".join(f"`{phrase}`" for phrase in phrases)) + elif title: + st.caption("No key phrases returned for this document.") + + with st.expander("Raw API response"): + st.json(payload) + + +def _show_message(message: tuple[str, str] | None) -> None: + """Render a stored error or warning. ``message`` is ``(level, text)``.""" + if not message: + return + level, detail = message + if level == "error": + st.error(detail) + return + st.warning(detail) + + +def _text_tab() -> None: + st.subheader("Sentiment and key phrases") + st.caption("Text is sent to your API, which calls Azure AI Language. Nothing is stored.") + + if "draft_text" not in st.session_state: + st.session_state.draft_text = "" + + def _load_example() -> None: + st.session_state.draft_text = _EXAMPLE_TEXT + + st.button("Load example review", on_click=_load_example) + + text = st.text_area( + "Text to analyze", + key="draft_text", + height=180, + placeholder="Paste a review, an email, or any short passage.", + ) + language = st.selectbox( + "Language", + options=["en", "es", "fr", "de", "it", "pt", "nl", "ja", "zh-Hans"], + index=0, + help="BCP-47 style code passed through to Azure AI Language.", + ) + + # Keep the last result in session state. Streamlit reruns the script on + # every widget change, and a message drawn only inside the button block + # disappears on the next run. + if st.button("Analyze text", type="primary"): + st.session_state.analysis_message = None + st.session_state.analysis_payload = None + if not text.strip(): + st.session_state.analysis_message = ("warning", "Enter some text first.") + else: + try: + st.session_state.analysis_payload = analyze_text([text], language=language) + except ApiConnectionError as exc: + st.session_state.analysis_message = ("error", str(exc)) + except ApiError as exc: + level = "warning" if exc.status_code == 503 else "error" + st.session_state.analysis_message = (level, exc.detail) + + _show_message(st.session_state.get("analysis_message")) + if st.session_state.get("analysis_payload"): + _render_analysis(st.session_state.analysis_payload) + + +def _ocr_tab() -> None: + st.subheader("Read text from an image") + st.caption("Images are sent to the API and read with Tesseract. Azure is not involved. 10 MB per file.") + + uploads = st.file_uploader( + "Images", + type=["png", "jpg", "jpeg", "tif", "tiff", "bmp", "webp", "gif"], + accept_multiple_files=True, + ) + if st.button("Extract text", type="primary"): + st.session_state.ocr_message = None + st.session_state.ocr_payload = None + if not uploads: + st.session_state.ocr_message = ("warning", "Choose at least one image.") + else: + files = [ + (upload.name, upload.getvalue(), upload.type or "application/octet-stream") + for upload in uploads + ] + try: + st.session_state.ocr_payload = extract_text(files) + except ApiConnectionError as exc: + st.session_state.ocr_message = ("error", str(exc)) + except ApiError as exc: + level = "warning" if exc.status_code == 503 else "error" + st.session_state.ocr_message = (level, exc.detail) + + _show_message(st.session_state.get("ocr_message")) + payload = st.session_state.get("ocr_payload") + if payload: + results = payload.get("results") or [] + elapsed = payload.get("time_taken_seconds") + if elapsed is not None: + st.caption(f"Finished in {elapsed}s.") + for index, item in enumerate(results): + upload = uploads[index] if uploads and index < len(uploads) else None + with st.container(border=True): + filename = item.get("filename") or (upload.name if upload is not None else "image") + st.markdown(f"**{filename}**") + preview, transcript = st.columns([1, 2]) + with preview: + if upload is not None: + st.image(upload, use_container_width=True) + with transcript: + text = item.get("text") or "" + if text.strip(): + state_key = f"ocr-{index}-{filename}" + # Seed once. Passing value= and a key on later reruns + # makes Streamlit reject the widget. + if state_key not in st.session_state: + st.session_state[state_key] = text + st.text_area("Extracted text", key=state_key, height=180) + else: + st.info("No text was found in this image.") + with st.expander("Raw API response"): + st.json(payload) + + +def main() -> None: + """Draw the workbench. Streamlit reruns this function on each interaction.""" + st.set_page_config( + page_title="Text & OCR Workbench", + page_icon="🔎", + layout="wide", + ) + # Streamlit's default warning yellow is about 4.2:1 on this page. + # Force alert copy to the same near-black as the rest of the UI. + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + st.title("Text & OCR Workbench") + st.write( + "Score sentiment, pull out key phrases, and read text out of images. " + "This page talks only to the local FastAPI service." + ) + _render_status() + text_tab, ocr_tab = st.tabs(["Analyze text", "Read an image"]) + with text_tab: + _text_tab() + with ocr_tab: + _ocr_tab() + + +main() diff --git a/main.py b/main.py index ec8f9cb..0eb4499 100644 --- a/main.py +++ b/main.py @@ -1,56 +1,11 @@ -import logging +"""Process entry point for ``uvicorn main:app`` and Gunicorn. -from fastapi import FastAPI -from opencensus.ext.azure.log_exporter import AzureLogHandler -from pydantic import BaseModel +The FastAPI instance lives in :mod:`app.main`. This module exists so the +Azure App Service startup command keeps working: -import utils +``gunicorn -w 2 -k uvicorn.workers.UvicornWorker main:app`` +""" -app = FastAPI() +from app.main import app -logger = logging.getLogger(__name__) -logger.setLevel(10) -logger.addHandler(AzureLogHandler( - connection_string='InstrumentationKey=ffe3b82f-1430-4ebb-ab39-34aa9a55f86b')) - -headers = { - "Ocp-Apim-Subscription-Key": '7b27544096db4f64a5d9e64963303ab0', - "Content-Type": "application/json", - "Accept": "application/json" -} - - -class Model(BaseModel): - text_to_analyze: list - - -@ app.post("/") -def analyze_text(text: Model): - response = {"sentiment": [], "keyphrases": []} - no_of_text = len(text.text_to_analyze) - tasks = [] - for i in range(no_of_text): - document = {"documents": [ - {"id": i + 1, "language": "en", "text": text.text_to_analyze[i]}]} - - sentiment = utils.call_text_analytics_api( - headers, document, endpoint='sentiment') - keyphrases = utils.call_text_analytics_api( - headers, document, endpoint='keyPhrases') - - log_data = { - "custom_dimensions": - { - "text": text.text_to_analyze[i], - "text_sentiment": sentiment["documents"][0]["sentiment"], - "text_keyphrases": keyphrases["documents"][0]["keyPhrases"] - } - } - logger.info('Text Processed Successfully', extra=log_data) - - response["sentiment"].append( - sentiment["documents"][0]) - response["keyphrases"].append( - keyphrases["documents"][0]) - - return response +__all__ = ["app"] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..78998da --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +pythonpath = . +testpaths = tests +addopts = -q diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..674765b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +# Local tests and the GitHub Actions test job. +-r requirements.txt +-r requirements-ui.txt +pytest==8.3.4 +httpx==0.28.1 diff --git a/requirements-ui.txt b/requirements-ui.txt new file mode 100644 index 0000000..8fba0e2 --- /dev/null +++ b/requirements-ui.txt @@ -0,0 +1,3 @@ +# Streamlit service. It calls the API over HTTP and does not import the API package. +streamlit==1.41.1 +requests==2.32.3 diff --git a/requirements.txt b/requirements.txt index 4164899..70ac894 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,65 +1,11 @@ -astroid==2.2.5 -autopep8==1.4.4 -azure-ai-textanalytics==5.2.0b3 -azure-common==1.1.28 -azure-core==1.23.0 -azure-identity==1.8.0 -bokeh==2.2.3 -cachetools==5.0.0 -certifi==2020.12.5 -cffi==1.15.0 -chardet==4.0.0 -charset-normalizer==2.0.12 -click==7.1.2 -cryptography==36.0.2 -fastapi==0.63.0 -google-api-core==2.7.1 -google-auth==2.6.2 -googleapis-common-protos==1.56.0 -gunicorn==20.0.4 -h11==0.12.0 -httptools==0.4.0 -idna==2.10 -isodate==0.6.0 -isort==4.3.21 -Jinja2==3.1.1 -lazy-object-proxy==1.4.2 -MarkupSafe==2.1.1 -mccabe==0.6.1 -msal==1.17.0 -msal-extensions==0.3.1 -msrest==0.6.21 -oauthlib==3.1.0 -opencensus==0.8.0 -opencensus-context==0.1.2 -opencensus-ext-azure==1.1.3 -packaging==20.8 -Pillow==9.1.0 -portalocker==2.4.0 -protobuf==3.20.0 -psutil==5.9.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pycodestyle==2.8.0 -pycparser==2.21 -pydantic==1.8.1 -PyJWT==2.3.0 -pylint==2.3.1 -pyparsing==3.0.7 -python-dateutil==2.8.2 -python-dotenv==0.20.0 -PyYAML==6.0 -requests==2.25.1 -requests-oauthlib==1.3.0 -rsa==4.8 -six==1.15.0 -starlette==0.13.6 -toml==0.10.2 -tornado==6.1 -typed-ast==1.4.0 -typing-extensions==4.1.1 -urllib3==1.26.4 -uvicorn==0.13.4 -uvloop==0.16.0 -wincertstore==0.2 -wrapt==1.14.0 +# API image and the Azure App Service deploy. The Streamlit UI has its own file. +fastapi==0.115.6 +uvicorn==0.34.0 +gunicorn==23.0.0 +pydantic==2.10.6 +python-multipart==0.0.20 +requests==2.32.3 +python-dotenv==1.0.1 +pytesseract==0.3.13 +Pillow==11.1.0 +azure-ai-textanalytics==5.3.0 diff --git a/fastapi-app/test.png b/samples/ocr-sample.png similarity index 100% rename from fastapi-app/test.png rename to samples/ocr-sample.png diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..52a8967 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,34 @@ +"""Fixtures shared by the API tests. + +Azure variables are cleared for every test so a developer's ``.env`` cannot +leak into the suite. Tests that need credentials set them explicitly. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +_AZURE_ENV_VARS = ( + "AZURE_LANGUAGE_ENDPOINT", + "AZURE_LANGUAGE_KEY", + "COG_SERVICE_ENDPOINT", + "COG_SERVICE_KEY", +) + + +@pytest.fixture(autouse=True) +def _clear_azure_env(monkeypatch: pytest.MonkeyPatch) -> None: + for name in _AZURE_ENV_VARS: + monkeypatch.delenv(name, raising=False) + + +@pytest.fixture +def client() -> Iterator[TestClient]: + """In-process API client. Entering the client runs the app lifespan.""" + with TestClient(app) as test_client: + yield test_client diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..62ae23b --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,235 @@ +"""HTTP behavior for health, text analysis, and OCR.""" + +from __future__ import annotations + +import io +import shutil +from typing import Any + +import pytest +from fastapi.testclient import TestClient +from PIL import Image, ImageDraw + +from app.config import MISSING_CREDENTIALS_MESSAGE, OCR_UNAVAILABLE_MESSAGE + + +def _png_bytes(text: str = "Hello OCR") -> bytes: + """A tiny PNG. OCR tests that mock Tesseract only need it to be a real image.""" + image = Image.new("RGB", (480, 120), "white") + draw = ImageDraw.Draw(image) + draw.text((16, 40), text, fill="black") + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +def _azure_post(url: str, headers: dict[str, str], json: dict[str, Any], timeout: int) -> Any: + """Stand-in for ``requests.post`` that echoes the documents it was given.""" + + class _Response: + status_code = 200 + + def json(self) -> dict[str, Any]: + documents = json["documents"] + if url.endswith("/sentiment"): + return { + "documents": [ + { + "id": document["id"], + "sentiment": "positive", + "confidenceScores": { + "positive": 0.9, + "neutral": 0.08, + "negative": 0.02, + }, + } + for document in documents + ], + "errors": [], + } + return { + "documents": [ + {"id": document["id"], "keyPhrases": ["royal hotel"]} + for document in documents + ], + "errors": [], + } + + assert headers["Ocp-Apim-Subscription-Key"] == "test-key" + assert timeout == 30 + return _Response() + + +def test_health(client: TestClient) -> None: + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_home_lists_routes(client: TestClient) -> None: + body = client.get("/").json() + assert "/api/v1/analyze" in body["analyze"] + assert body["ocr"] == "POST /api/v1/extract_text" + + +def test_status_when_azure_is_not_configured(client: TestClient) -> None: + body = client.get("/api/v1/status").json() + assert body["azure_language_configured"] is False + assert any("not configured" in message for message in body["messages"]) + assert "test-key" not in response_text(body) + + +def response_text(body: dict[str, Any]) -> str: + return " ".join(body["messages"]) + + +def test_analyze_without_credentials_is_a_clear_503(client: TestClient) -> None: + response = client.post("/api/v1/analyze", json={"text_to_analyze": ["A lovely stay."]}) + assert response.status_code == 503 + assert response.json()["detail"] == MISSING_CREDENTIALS_MESSAGE + + +def test_legacy_post_root_uses_the_same_credential_check(client: TestClient) -> None: + response = client.post("/", json={"text_to_analyze": ["A lovely stay."]}) + assert response.status_code == 503 + assert "AZURE_LANGUAGE_KEY" in response.json()["detail"] + + +def test_placeholder_credentials_are_treated_as_missing( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv( + "AZURE_LANGUAGE_ENDPOINT", + "https://your-resource.cognitiveservices.azure.com", + ) + monkeypatch.setenv("AZURE_LANGUAGE_KEY", "your-key-here") + response = client.post("/api/v1/analyze", json={"text_to_analyze": ["Hello"]}) + assert response.status_code == 503 + + +def test_analyze_rejects_blank_text(client: TestClient) -> None: + response = client.post("/api/v1/analyze", json={"text_to_analyze": [" "]}) + assert response.status_code == 422 + + +def test_analyze_with_credentials_shapes_the_response( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("AZURE_LANGUAGE_ENDPOINT", "https://contoso.cognitiveservices.azure.com") + monkeypatch.setenv("AZURE_LANGUAGE_KEY", "test-key") + monkeypatch.setattr("app.text_analytics.requests.post", _azure_post) + + response = client.post( + "/api/v1/analyze", + json={"text_to_analyze": ["The Royal Hotel was wonderful."], "language": "en"}, + ) + assert response.status_code == 200 + body = response.json() + assert body["results"][0]["sentiment"] == "positive" + assert body["results"][0]["key_phrases"] == ["royal hotel"] + assert body["sentiment"][0]["sentiment"] == "positive" + assert body["keyphrases"][0]["keyPhrases"] == ["royal hotel"] + + +def test_analyze_batches_more_than_ten_documents( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[str] = [] + + def counting_post(url: str, headers: dict[str, str], json: dict[str, Any], timeout: int) -> Any: + calls.append(url) + return _azure_post(url, headers, json, timeout) + + monkeypatch.setenv("COG_SERVICE_ENDPOINT", "https://legacy.cognitiveservices.azure.com/") + monkeypatch.setenv("COG_SERVICE_KEY", "test-key") + monkeypatch.setattr("app.text_analytics.requests.post", counting_post) + + texts = [f"Review number {index}" for index in range(11)] + response = client.post("/api/v1/analyze", json={"text_to_analyze": texts}) + assert response.status_code == 200 + assert len(response.json()["results"]) == 11 + # Two batches, and each batch calls sentiment and key phrases. + assert len(calls) == 4 + assert calls[0].endswith("/text/analytics/v3.1/sentiment") + + +def test_analyze_reports_rejected_credentials( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + class _Denied: + status_code = 401 + + def json(self) -> dict[str, str]: + return {"error": {"message": "Access denied due to invalid subscription key"}} + + monkeypatch.setenv("AZURE_LANGUAGE_ENDPOINT", "https://contoso.cognitiveservices.azure.com") + monkeypatch.setenv("AZURE_LANGUAGE_KEY", "test-key") + monkeypatch.setattr("app.text_analytics.requests.post", lambda *args, **kwargs: _Denied()) + + response = client.post("/api/v1/analyze", json={"text_to_analyze": ["Hello"]}) + assert response.status_code == 502 + detail = response.json()["detail"] + assert "rejected" in detail + assert "test-key" not in detail + + +def test_ocr_when_tesseract_is_missing(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("app.ocr.tesseract_available", lambda: False) + response = client.post( + "/api/v1/extract_text", + files=[("images", ("note.png", _png_bytes(), "image/png"))], + ) + assert response.status_code == 503 + assert response.json()["detail"] == OCR_UNAVAILABLE_MESSAGE + + +def test_ocr_rejects_a_non_image(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("app.ocr.tesseract_available", lambda: True) + response = client.post( + "/api/v1/extract_text", + files=[("images", ("notes.txt", b"not an image", "text/plain"))], + ) + assert response.status_code == 400 + assert "notes.txt" in response.json()["detail"] + + +def test_ocr_rejects_an_oversized_upload(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("app.ocr.tesseract_available", lambda: True) + monkeypatch.setattr("app.ocr.MAX_IMAGE_BYTES", 8) + response = client.post( + "/api/v1/extract_text", + files=[("images", ("big.png", b"0123456789", "image/png"))], + ) + assert response.status_code == 400 + assert "10 MB" in response.json()["detail"] or "smaller" in response.json()["detail"] + + +def test_ocr_returns_text_from_tesseract(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("app.ocr.tesseract_available", lambda: True) + monkeypatch.setattr("app.ocr.pytesseract.image_to_string", lambda image, lang="eng": "Hello from Tesseract") + + response = client.post( + "/api/v1/extract_text", + files=[("images", ("board.png", _png_bytes(), "image/png"))], + ) + assert response.status_code == 200 + body = response.json() + assert body["results"][0] == {"filename": "board.png", "text": "Hello from Tesseract"} + assert body["board.png"] == "Hello from Tesseract" + assert "Time Taken" in body + assert body["time_taken_seconds"] == body["Time Taken"] + + +@pytest.mark.skipif(shutil.which("tesseract") is None, reason="tesseract binary is not installed") +def test_ocr_reads_the_sample_image(client: TestClient) -> None: + sample = open("samples/ocr-sample.png", "rb") + try: + response = client.post( + "/api/v1/extract_text", + files=[("images", ("ocr-sample.png", sample.read(), "image/png"))], + ) + finally: + sample.close() + assert response.status_code == 200 + text = response.json()["results"][0]["text"] + assert "quick brown" in text.lower() diff --git a/tests/test_api_client.py b/tests/test_api_client.py new file mode 100644 index 0000000..0ce3e32 --- /dev/null +++ b/tests/test_api_client.py @@ -0,0 +1,70 @@ +"""The UI's HTTP client, without starting Streamlit.""" + +from __future__ import annotations + +import py_compile +from typing import Any + +import pytest + +from frontend.api_client import ApiConnectionError, ApiError, analyze_text, format_detail, get_status + + +def test_format_detail_reads_a_string() -> None: + assert format_detail({"detail": "Azure Language credentials are not configured."}, "fallback") == ( + "Azure Language credentials are not configured." + ) + + +def test_format_detail_flattens_validation_errors() -> None: + payload = { + "detail": [ + {"loc": ["body", "text_to_analyze"], "msg": "Field required", "type": "missing"}, + ] + } + assert format_detail(payload, "fallback") == "text_to_analyze: Field required" + + +def test_format_detail_uses_the_fallback() -> None: + assert format_detail({"unexpected": True}, "fallback") == "fallback" + + +def test_get_status_returns_the_payload(monkeypatch: pytest.MonkeyPatch) -> None: + class _Response: + status_code = 200 + + def json(self) -> dict[str, Any]: + return {"azure_language_configured": False, "ocr_available": True, "messages": ["ok"]} + + monkeypatch.setattr("frontend.api_client.requests.request", lambda *args, **kwargs: _Response()) + assert get_status()["ocr_available"] is True + + +def test_analyze_text_surfaces_503(monkeypatch: pytest.MonkeyPatch) -> None: + class _Response: + status_code = 503 + + def json(self) -> dict[str, str]: + return {"detail": "Azure Language credentials are not configured."} + + monkeypatch.setattr("frontend.api_client.requests.request", lambda *args, **kwargs: _Response()) + with pytest.raises(ApiError) as caught: + analyze_text(["Hello"]) + assert caught.value.status_code == 503 + assert "not configured" in caught.value.detail + + +def test_connection_error_names_the_base_url(monkeypatch: pytest.MonkeyPatch) -> None: + import requests + + def _boom(*args: object, **kwargs: object) -> None: + raise requests.ConnectionError("refused") + + monkeypatch.setattr("frontend.api_client.requests.request", _boom) + with pytest.raises(ApiConnectionError) as caught: + get_status() + assert "localhost:8000" in str(caught.value) + + +def test_streamlit_app_compiles() -> None: + py_compile.compile("frontend/streamlit_app.py", doraise=True) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..8654bd8 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,29 @@ +"""The batch review script exits clearly when Azure is not configured.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + + +def test_cli_exits_when_credentials_are_missing() -> None: + env = os.environ.copy() + # Empty strings count as "set", so python-dotenv will not replace them from a local .env. + env["AZURE_LANGUAGE_ENDPOINT"] = "" + env["AZURE_LANGUAGE_KEY"] = "" + env["COG_SERVICE_ENDPOINT"] = "" + env["COG_SERVICE_KEY"] = "" + + script = Path("text-analytics-sdk/text-analysis.py") + completed = subprocess.run( + [sys.executable, str(script)], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert completed.returncode == 1 + assert "not configured" in completed.stdout + assert "AZURE_LANGUAGE_KEY" in completed.stdout diff --git a/tests/test_secrets.py b/tests/test_secrets.py new file mode 100644 index 0000000..d4171cf --- /dev/null +++ b/tests/test_secrets.py @@ -0,0 +1,25 @@ +"""Guardrails so credentials do not land in the tree again.""" + +from __future__ import annotations + +from pathlib import Path + +# Patterns that marked the old hardcoded Azure wiring. Split so this file +# does not contain the strings it is looking for. The secret values +# themselves are intentionally not repeated here. +_FORBIDDEN = ( + "Instrumentation" + "Key=", + "language-project" + ".cognitiveservices.azure.com", +) + + +def test_python_sources_do_not_embed_the_old_azure_wiring() -> None: + offenders: list[str] = [] + for path in Path(".").rglob("*.py"): + if any(part in {".venv", "venv", ".git", "__pycache__"} for part in path.parts): + continue + text = path.read_text(encoding="utf8") + for pattern in _FORBIDDEN: + if pattern in text: + offenders.append(f"{path}: {pattern}") + assert offenders == [] diff --git a/tests/test_streamlit_app.py b/tests/test_streamlit_app.py new file mode 100644 index 0000000..79866a2 --- /dev/null +++ b/tests/test_streamlit_app.py @@ -0,0 +1,65 @@ +"""Streamlit page behavior, driven with Streamlit's AppTest harness. + +These tests do not open a browser. They run the script, set widgets, and +read the elements the page would show. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from streamlit.testing.v1 import AppTest + + +def _response(status_code: int, payload: dict[str, Any]) -> Any: + class _Response: + def __init__(self) -> None: + self.status_code = status_code + + def json(self) -> dict[str, Any]: + return payload + + return _Response() + + +def test_ui_explains_when_the_api_is_down(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("API_BASE_URL", "http://127.0.0.1:9") + app = AppTest.from_file("frontend/streamlit_app.py") + app.run(timeout=30) + assert not app.exception + assert any("Could not reach the API" in item.value for item in app.error) + assert any(item.value == "Text & OCR Workbench" for item in app.title) + + +def test_ui_shows_the_azure_setup_message(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_request(method: str, url: str, timeout: int, **kwargs: Any) -> Any: + if url.endswith("/api/v1/status"): + return _response( + 200, + { + "azure_language_configured": False, + "ocr_available": True, + "messages": [ + "Azure Language credentials are not configured. Set AZURE_LANGUAGE_KEY.", + "Tesseract OCR is available.", + ], + }, + ) + return _response( + 503, + {"detail": "Azure Language credentials are not configured. Set AZURE_LANGUAGE_KEY."}, + ) + + monkeypatch.setattr("frontend.api_client.requests.request", fake_request) + app = AppTest.from_file("frontend/streamlit_app.py") + app.run(timeout=30) + assert not app.exception + assert any("not configured" in item.value for item in app.warning) + + text_area = next(item for item in app.text_area if item.label == "Text to analyze") + text_area.set_value("A lovely stay.").run() + analyze = next(item for item in app.button if item.label == "Analyze text") + analyze.click().run() + assert not app.exception + assert any("AZURE_LANGUAGE_KEY" in item.value for item in app.warning) diff --git a/text-analytics-sdk/text-analysis.py b/text-analytics-sdk/text-analysis.py index 6f1d1f5..781b0c0 100644 --- a/text-analytics-sdk/text-analysis.py +++ b/text-analytics-sdk/text-analysis.py @@ -1,74 +1,115 @@ +"""Batch text analytics for the sample hotel reviews. + +Run from anywhere: + + python text-analytics-sdk/text-analysis.py + +Credentials come from the environment (or a ``.env`` file in the current +working directory). When they are missing, the script prints how to set them +and exits with status 1. It does not call Azure in that case. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + from dotenv import load_dotenv -import os -# Import namespaces -from azure.core.credentials import AzureKeyCredential +# The script lives in a subfolder; the shared config module lives at the repo root. +_ROOT = Path(__file__).resolve().parents[1] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + from azure.ai.textanalytics import TextAnalyticsClient +from azure.core.credentials import AzureKeyCredential + +from app.config import MISSING_CREDENTIALS_MESSAGE, load_settings + +_REVIEWS_DIR = Path(__file__).resolve().parent / "reviews" + + +def _load_documents() -> list[dict[str, str]]: + documents: list[dict[str, str]] = [] + if not _REVIEWS_DIR.is_dir(): + return documents + for file_path in sorted(_REVIEWS_DIR.glob("*.txt")): + text = file_path.read_text(encoding="utf8") + documents.append({"id": file_path.name, "text": text}) + return documents + + +def main() -> int: + """Analyze every review file. Returns a process exit code.""" + load_dotenv() + settings = load_settings() + if not settings.azure_language_configured: + print(MISSING_CREDENTIALS_MESSAGE) + return 1 + + documents = _load_documents() + if not documents: + print(f"No review files found in {_REVIEWS_DIR}.") + return 1 + # load_settings() only returns configured when both values are present. + assert settings.azure_language_endpoint is not None + assert settings.azure_language_key is not None + client = TextAnalyticsClient( + endpoint=settings.azure_language_endpoint, + credential=AzureKeyCredential(settings.azure_language_key), + ) -def main(): try: - # Load Configuration Settings - load_dotenv() - cog_endpoint = os.getenv('COG_SERVICE_ENDPOINT') - cog_key = os.getenv('COG_SERVICE_KEY') - - # Create Text Analytics client - credential = AzureKeyCredential(cog_key) - cog_client = TextAnalyticsClient(endpoint=cog_endpoint, credential=credential) - - # Read all text files from the reviews folder - reviews_folder = 'reviews' - documents = [] - - for file_name in os.listdir(reviews_folder): - file_path = os.path.join(reviews_folder, file_name) - with open(file_path, encoding='utf8') as f: - text = f.read() - documents.append({"id": file_name, "text": text}) # Each file is a document - - if not documents: - print("No documents found in the reviews folder.") - exit() - - # Batch API Calls print("\nProcessing batch request...\n") - # Language Detection - language_results = cog_client.detect_language(documents=documents) + language_results = client.detect_language(documents=documents) for doc, result in zip(documents, language_results): + if result.is_error: + print(f"\nFile: {doc['id']}\nLanguage error: {result.error}") + continue print(f"\nFile: {doc['id']}\nLanguage: {result.primary_language.name}") - # Sentiment Analysis - sentiment_results = cog_client.analyze_sentiment(documents=documents) + sentiment_results = client.analyze_sentiment(documents=documents) for doc, result in zip(documents, sentiment_results): + if result.is_error: + print(f"\nFile: {doc['id']}\nSentiment error: {result.error}") + continue print(f"\nFile: {doc['id']}\nSentiment: {result.sentiment}") - # Key Phrase Extraction - key_phrase_results = cog_client.extract_key_phrases(documents=documents) + key_phrase_results = client.extract_key_phrases(documents=documents) for doc, result in zip(documents, key_phrase_results): + if result.is_error: + print(f"\nFile: {doc['id']}\nKey phrase error: {result.error}") + continue if result.key_phrases: print(f"\nFile: {doc['id']}\nKey Phrases: {', '.join(result.key_phrases)}") - # Named Entity Recognition - entity_results = cog_client.recognize_entities(documents=documents) + entity_results = client.recognize_entities(documents=documents) for doc, result in zip(documents, entity_results): + if result.is_error: + print(f"\nFile: {doc['id']}\nEntity error: {result.error}") + continue if result.entities: print(f"\nFile: {doc['id']}\nEntities:") for entity in result.entities: print(f"\t{entity.text} ({entity.category})") - # Linked Entity Recognition - linked_entity_results = cog_client.recognize_linked_entities(documents=documents) + linked_entity_results = client.recognize_linked_entities(documents=documents) for doc, result in zip(documents, linked_entity_results): + if result.is_error: + print(f"\nFile: {doc['id']}\nLinked entity error: {result.error}") + continue if result.entities: print(f"\nFile: {doc['id']}\nLinked Entities:") for entity in result.entities: print(f"\t{entity.name} ({entity.url})") + except Exception as exc: + print(f"Error talking to Azure AI Language: {exc}") + return 1 - except Exception as ex: - print(f"Error: {ex}") + return 0 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/utils.py b/utils.py deleted file mode 100644 index 03ab29a..0000000 --- a/utils.py +++ /dev/null @@ -1,7 +0,0 @@ -import requests as req - - -def call_text_analytics_api(headers, document, endpoint): - response = req.post("https://language-project.cognitiveservices.azure.com/text/analytics/v3.1/" + - endpoint, headers=headers, json=document) - return response.json()