chore(infra): add AI flyer parsing configuration and retry logic
- Add FLYER_AI_TIMEOUT_MS and FLYER_AI_RETRIES environment variables - Configure timeout and retry settings in compose.yml - Update AiFlyerParserService with configurable timeout and retry logic - Add text window reduction strategy for retry attempts - Update documentation in TEKNISK_BESKRIVNING.md - Fix ESLint configuration in app.security.spec.ts
This commit is contained in:
@@ -19,3 +19,5 @@ SEED_USER2_PASSWORD=Test-Anv2-FBG
|
||||
AUTH_SECRET=WheqAss4F/al9yRZRqepJEBs6TzPsN3brX0iBiF4Oww=
|
||||
JWT_SECRET=uK9yRQpyyWOcHYcYbpAdsJ7NJcEsyCYZcgF82OnBz2k=
|
||||
MISTRAL_API_KEY=JGPjLuNnzaLSYMxKbexLZohUOegrSLye
|
||||
FLYER_AI_TIMEOUT_MS=45000
|
||||
FLYER_AI_RETRIES=2
|
||||
|
||||
@@ -19,6 +19,8 @@ JWT_SECRET=byt-ut-mig
|
||||
# Mistral AI
|
||||
# Hämtas från: https://console.mistral.ai/
|
||||
MISTRAL_API_KEY=
|
||||
FLYER_AI_TIMEOUT_MS=45000
|
||||
FLYER_AI_RETRIES=2
|
||||
|
||||
# Publik URL (används av frontend)
|
||||
NEXT_PUBLIC_APP_URL=https://recept.gynther.se
|
||||
|
||||
@@ -18,6 +18,11 @@ Se även: README.md för användarflöde, och AI-FUNKTIONER.md för AI-detaljer.
|
||||
|
||||
# Nyheter och förbättringar (2026-05-18)
|
||||
|
||||
- **Flyerimport intern i recipe-api:** `/api/flyer-import/parse` använder nu en intern pipeline i backend (`TextExtractorService` + `AiFlyerParserService` + `FlyerNormalizerService`) och är inte längre beroende av `importer-api`.
|
||||
- **Textutvinning för flyer:** PDF tolkas primärt med `pdf-parse`; vid bildfiler eller skannad PDF-fallback används OCR via `tesseract.js`.
|
||||
- **AI-parse för flyer:** Mistral Tiny används för strukturerad extraktion av flyer-rader till JSON, följt av normalisering av pris/enhet/kategori.
|
||||
- **Timeout/retry-härdning för flyer-AI:** `AiFlyerParserService` har konfigurerbar timeout (`FLYER_AI_TIMEOUT_MS`, default 30000 ms) och retry med successivt kortare textfönster (`FLYER_AI_RETRIES`, default 2) för att minska 503 vid långsamma modellanrop.
|
||||
|
||||
- **Backend linting i CI:** ESLint är infört för backend (`backend/eslint.config.mjs`, `npm run lint`) och körs i `.github/workflows/test.yml`.
|
||||
- **Flutter lint-konfiguration:** `flutter/analysis_options.yaml` är tillagd och inkluderar `package:flutter_lints/flutter.yaml`.
|
||||
- **Prisma query logging (miljöstyrd):** `PrismaService` konfigurerar loggnivåer via env-variabeln `PRISMA_LOG_QUERIES`.
|
||||
|
||||
@@ -7,7 +7,7 @@ import { RolesGuard } from './auth/roles.guard';
|
||||
describe('App security configuration', () => {
|
||||
function getAppModuleClass() {
|
||||
process.env.JWT_SECRET = process.env.JWT_SECRET ?? 'test-secret';
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
// eslint-disable-next-line global-require
|
||||
return require('./app.module').AppModule as any;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,8 @@ export interface AiFlyerParseResult {
|
||||
@Injectable()
|
||||
export class AiFlyerParserService {
|
||||
private readonly logger = new Logger(AiFlyerParserService.name);
|
||||
private readonly timeoutMs = 15_000;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly maxRetries: number;
|
||||
private mistral: any;
|
||||
private apiKey: string;
|
||||
|
||||
@@ -30,6 +31,9 @@ export class AiFlyerParserService {
|
||||
if (!this.apiKey) {
|
||||
throw new Error('MISTRAL_API_KEY environment variable not set');
|
||||
}
|
||||
|
||||
this.timeoutMs = this.readPositiveIntEnv('FLYER_AI_TIMEOUT_MS', 30_000);
|
||||
this.maxRetries = this.readPositiveIntEnv('FLYER_AI_RETRIES', 2);
|
||||
}
|
||||
|
||||
private async getClient(): Promise<any> {
|
||||
@@ -50,12 +54,21 @@ export class AiFlyerParserService {
|
||||
throw new BadRequestException('Flyer-texten är tom. Kan inte fortsätta.');
|
||||
}
|
||||
|
||||
const prompt = this.buildPrompt(text);
|
||||
try {
|
||||
const client = await this.getClient();
|
||||
const textWindows = [5000, 3000, 2000];
|
||||
const attempts = Math.max(1, Math.min(this.maxRetries + 1, textWindows.length));
|
||||
let lastError: unknown = null;
|
||||
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
const window = textWindows[i];
|
||||
const prompt = this.buildPrompt(text, window);
|
||||
|
||||
try {
|
||||
this.logger.debug('Sending request to Mistral Tiny');
|
||||
this.logger.debug(
|
||||
`Sending request to Mistral Tiny (attempt ${i + 1}/${attempts}, timeout=${this.timeoutMs}ms, textWindow=${window})`,
|
||||
);
|
||||
|
||||
const client = await this.getClient();
|
||||
const response = await this.withTimeout<any>(
|
||||
client.chat({
|
||||
model: 'mistral-tiny',
|
||||
@@ -73,7 +86,6 @@ export class AiFlyerParserService {
|
||||
|
||||
this.logger.debug(`Mistral response length: ${content.length} chars`);
|
||||
|
||||
// Rensa och parse JSON
|
||||
const jsonString = this.sanitizeJsonResponse(content);
|
||||
const items = JSON.parse(jsonString) as Array<Record<string, unknown>>;
|
||||
|
||||
@@ -82,6 +94,18 @@ export class AiFlyerParserService {
|
||||
}
|
||||
|
||||
return items.map((item, idx) => this.normalizeAiItem(item, idx));
|
||||
} catch (attemptErr) {
|
||||
lastError = attemptErr;
|
||||
if (!this.isRetryableError(attemptErr) || i === attempts - 1) {
|
||||
throw attemptErr;
|
||||
}
|
||||
this.logger.warn(
|
||||
`Mistral attempt ${i + 1} failed (${this.toErrorMessage(attemptErr)}). Retrying with shorter text window.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new ServiceUnavailableException('AI-anrop misslyckades');
|
||||
} catch (err) {
|
||||
if (err instanceof SyntaxError) {
|
||||
this.logger.error(`JSON parse error: ${String(err)}`);
|
||||
@@ -121,9 +145,8 @@ export class AiFlyerParserService {
|
||||
/**
|
||||
* Bygger systemprompten för Mistral.
|
||||
*/
|
||||
private buildPrompt(text: string): string {
|
||||
// Trunkera långt text för att spara tokens
|
||||
const truncatedText = text.length > 5000 ? text.substring(0, 5000) : text;
|
||||
private buildPrompt(text: string, maxTextLength: number): string {
|
||||
const truncatedText = text.length > maxTextLength ? text.substring(0, maxTextLength) : text;
|
||||
|
||||
return `Du är en expert på att tolka svenska matvaruflyers (t.ex. från Willys, Coop, ICA).
|
||||
|
||||
@@ -231,4 +254,32 @@ Exempel på utdata:
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
private readPositiveIntEnv(key: string, fallback: number): number {
|
||||
const raw = process.env[key];
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
this.logger.warn(`Invalid ${key} value: "${raw}". Falling back to ${fallback}.`);
|
||||
return fallback;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
private isRetryableError(err: unknown): boolean {
|
||||
if (err instanceof ServiceUnavailableException) return true;
|
||||
const message = this.toErrorMessage(err).toLowerCase();
|
||||
return (
|
||||
message.includes('timeout') ||
|
||||
message.includes('timed out') ||
|
||||
message.includes('rate limit') ||
|
||||
message.includes('econnreset') ||
|
||||
message.includes('socket hang up')
|
||||
);
|
||||
}
|
||||
|
||||
private toErrorMessage(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
return String(err);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ services:
|
||||
NODE_ENV: "production"
|
||||
DATABASE_URL: "mysql://root:${MARIADB_ROOT_PASSWORD}@recipe-db:3306/${MARIADB_DATABASE}"
|
||||
MISTRAL_API_KEY: "${MISTRAL_API_KEY:-}"
|
||||
FLYER_AI_TIMEOUT_MS: "${FLYER_AI_TIMEOUT_MS:-30000}"
|
||||
FLYER_AI_RETRIES: "${FLYER_AI_RETRIES:-2}"
|
||||
JWT_SECRET: "${JWT_SECRET}"
|
||||
ALLOWED_ORIGIN: "${NEXT_PUBLIC_APP_URL}"
|
||||
ADMIN_NADMIN_PASSWORD: "${ADMIN_NADMIN_PASSWORD}"
|
||||
|
||||
Reference in New Issue
Block a user