refactor(ai): enhance AI trace integration and OCR normalization
- Add FlyerTraceSupplement type for AI trace metadata - Implement getFlyerTraceSupplements method to fetch trace supplements - Update AiTraceService to include prompt/rawOutput and counters in flyer traces - Add persistFlyerTrace method to FlyerImportService for trace persistence - Enhance AiFlyerParserService to return structured trace data with prompts and retries - Update FlyerNormalizerService with OCR typo fixes for cheese variants and spröd bakad firre - Improve Flutter admin panel with selectable text, warnings display, and tooltips - Add comprehensive tests for AI trace supplements and normalization rules
This commit is contained in:
Nils-Johan Gynther
@@ -90,4 +90,59 @@ describe('AiTraceService receipt masking', () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns flyer prompt/rawOutput and trace counters from aiTrace supplement', async () => {
|
||||
prismaMock.flyerSession.findUnique.mockResolvedValue({
|
||||
id: 101,
|
||||
userId: 7,
|
||||
createdAt: new Date('2026-05-21T12:00:00.000Z'),
|
||||
sourceFileName: 'willys.pdf',
|
||||
sourceMimeType: 'application/pdf',
|
||||
sourceFileSize: 12345,
|
||||
user: { username: 'admin', email: 'admin@example.com' },
|
||||
items: [
|
||||
{
|
||||
id: 1,
|
||||
rawName: 'Tomat',
|
||||
normalizedName: 'tomat',
|
||||
brand: null,
|
||||
categoryHint: 'Grönsaker',
|
||||
categoryId: null,
|
||||
price: null,
|
||||
priceUnit: null,
|
||||
comparisonPrice: null,
|
||||
comparisonUnit: null,
|
||||
weight: null,
|
||||
bundleWeight: null,
|
||||
isBundle: false,
|
||||
bundleItems: [],
|
||||
offerText: null,
|
||||
parseConfidence: 0.9,
|
||||
parseReasons: ['low_confidence'],
|
||||
matchedProductId: null,
|
||||
matchedProductName: null,
|
||||
matchedVia: 'none',
|
||||
matchConfidence: null,
|
||||
matchReasons: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
prismaMock.aiTrace.findMany.mockResolvedValue([
|
||||
{
|
||||
sessionId: 101,
|
||||
prompt: 'Flyer prompt med email kund@example.com',
|
||||
rawOutput: '{"ok":true}',
|
||||
normalizedOutput: { retryCount: 2, chunkCount: 4 },
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await service.getTraceById('flyer-101');
|
||||
|
||||
expect(result.prompt).toContain('[MASKED]');
|
||||
expect(result.rawOutput).toContain('{"ok":true}');
|
||||
expect(result.retryCount).toBe(2);
|
||||
expect(result.chunkCount).toBe(4);
|
||||
expect(result.warnings).toContain('parse:low_confidence');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -53,6 +53,13 @@ export type AiTraceDetail = {
|
||||
summary: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type FlyerTraceSupplement = {
|
||||
prompt: string | null;
|
||||
rawOutput: string | null;
|
||||
retryCount: number | null;
|
||||
chunkCount: number | null;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class AiTraceService {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
@@ -124,8 +131,26 @@ export class AiTraceService {
|
||||
};
|
||||
});
|
||||
|
||||
const supplements = await this.getFlyerTraceSupplements(page.map((session) => session.id));
|
||||
|
||||
const withSupplements = items.map((item) => {
|
||||
const sessionId = item.sessionId ?? 0;
|
||||
const supplement = supplements.get(sessionId);
|
||||
const hasPrompt = item.hasPrompt || !!supplement?.prompt;
|
||||
const hasOutput = item.hasOutput || !!supplement?.rawOutput;
|
||||
const error = item.status === 'warning' && item.warningsCount > 0
|
||||
? `Det finns ${item.warningsCount} varningar i detaljvyn.`
|
||||
: item.error;
|
||||
return {
|
||||
...item,
|
||||
hasPrompt,
|
||||
hasOutput,
|
||||
error,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
items,
|
||||
items: withSupplements,
|
||||
nextCursor: hasMore ? String(page[page.length - 1]?.id ?? '') : null,
|
||||
};
|
||||
}
|
||||
@@ -182,6 +207,7 @@ export class AiTraceService {
|
||||
|
||||
const warnings = this.collectWarnings(session.items);
|
||||
const status = this.statusFromSession(session.items.length, warnings.length);
|
||||
const supplement = await this.getFlyerTraceSupplementBySessionId(session.id);
|
||||
|
||||
const normalizedOutput = {
|
||||
sessionId: session.id,
|
||||
@@ -228,19 +254,20 @@ export class AiTraceService {
|
||||
fileName: session.sourceFileName,
|
||||
model: 'ministral-8b-2512',
|
||||
durationMs: null,
|
||||
retryCount: null,
|
||||
chunkCount: null,
|
||||
retryCount: supplement.retryCount,
|
||||
chunkCount: supplement.chunkCount,
|
||||
warnings,
|
||||
error: session.items.length === 0 ? 'Inga produkter kunde extraheras från flyern.' : null,
|
||||
prompt: null,
|
||||
rawOutput: JSON.stringify(this.maskSensitiveData(normalizedOutput)),
|
||||
prompt: supplement.prompt,
|
||||
rawOutput:
|
||||
this.maskRawOutput(supplement.rawOutput) ?? JSON.stringify(this.maskSensitiveData(normalizedOutput)),
|
||||
normalizedOutput: this.maskSensitiveData(normalizedOutput),
|
||||
summary: {
|
||||
source: 'flyer',
|
||||
sessionId: session.id,
|
||||
itemCount: session.items.length,
|
||||
warningsCount: warnings.length,
|
||||
promptAvailable: false,
|
||||
promptAvailable: !!supplement.prompt,
|
||||
outputAvailable: true,
|
||||
retentionHintDays: 30,
|
||||
maskedFields: AI_TRACE_MASK_FIELDS,
|
||||
@@ -248,6 +275,57 @@ export class AiTraceService {
|
||||
};
|
||||
}
|
||||
|
||||
private async getFlyerTraceSupplements(sessionIds: number[]): Promise<Map<number, FlyerTraceSupplement>> {
|
||||
if (sessionIds.length === 0) return new Map<number, FlyerTraceSupplement>();
|
||||
|
||||
const rows = await this.prisma.aiTrace.findMany({
|
||||
where: {
|
||||
source: 'flyer',
|
||||
sessionId: { in: sessionIds },
|
||||
},
|
||||
orderBy: [{ sessionId: 'desc' }, { createdAt: 'desc' }],
|
||||
select: {
|
||||
sessionId: true,
|
||||
prompt: true,
|
||||
rawOutput: true,
|
||||
normalizedOutput: true,
|
||||
},
|
||||
});
|
||||
|
||||
const out = new Map<number, FlyerTraceSupplement>();
|
||||
for (const row of rows) {
|
||||
if (row.sessionId == null || out.has(row.sessionId)) continue;
|
||||
out.set(row.sessionId, {
|
||||
prompt: row.prompt ? this.maskSensitiveText(row.prompt) : null,
|
||||
rawOutput: row.rawOutput,
|
||||
retryCount: this.extractTraceNumber(row.normalizedOutput, 'retryCount'),
|
||||
chunkCount: this.extractTraceNumber(row.normalizedOutput, 'chunkCount'),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private async getFlyerTraceSupplementBySessionId(sessionId: number): Promise<FlyerTraceSupplement> {
|
||||
const rows = await this.getFlyerTraceSupplements([sessionId]);
|
||||
return rows.get(sessionId) ?? {
|
||||
prompt: null,
|
||||
rawOutput: null,
|
||||
retryCount: null,
|
||||
chunkCount: null,
|
||||
};
|
||||
}
|
||||
|
||||
private extractTraceNumber(value: unknown, key: string): number | null {
|
||||
if (!value || typeof value !== 'object') return null;
|
||||
const entry = (value as Record<string, unknown>)[key];
|
||||
if (typeof entry === 'number' && Number.isFinite(entry)) return entry;
|
||||
if (typeof entry === 'string') {
|
||||
const parsed = Number.parseInt(entry, 10);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private statusFromSession(itemCount: number, warningsCount: number): AiTraceStatus {
|
||||
if (itemCount <= 0) return 'error';
|
||||
if (warningsCount > 0) return 'warning';
|
||||
|
||||
@@ -41,6 +41,12 @@ type FlyerParseResponse = {
|
||||
parserVersion: 'v1';
|
||||
items: FlyerParseItem[];
|
||||
warnings: string[];
|
||||
trace: {
|
||||
prompt: string | null;
|
||||
rawOutput: string | null;
|
||||
chunkCount: number | null;
|
||||
retryCount: number | null;
|
||||
};
|
||||
};
|
||||
|
||||
type ExtractedOfferSignals = {
|
||||
@@ -71,6 +77,7 @@ export class FlyerImportService {
|
||||
) {}
|
||||
|
||||
async parseAndMatch(file: Express.Multer.File, userId: number): Promise<FlyerImportResponse> {
|
||||
const startedAt = Date.now();
|
||||
const parsed = await this.parseViaInternal(file);
|
||||
|
||||
const [products, aliases] = await Promise.all([
|
||||
@@ -138,6 +145,24 @@ export class FlyerImportService {
|
||||
|
||||
const persistedItems = await this.persistSessionWithItems(userId, parsed.retailer, items, file);
|
||||
|
||||
await this.persistFlyerTrace({
|
||||
userId,
|
||||
sessionId: persistedItems.sessionId,
|
||||
model: 'ministral-8b-2512',
|
||||
prompt: parsed.trace.prompt,
|
||||
rawOutput: parsed.trace.rawOutput,
|
||||
normalizedOutput: {
|
||||
sessionId: persistedItems.sessionId,
|
||||
warnings: parsed.warnings,
|
||||
itemCount: persistedItems.items.length,
|
||||
chunkCount: parsed.trace.chunkCount,
|
||||
retryCount: parsed.trace.retryCount,
|
||||
},
|
||||
status: persistedItems.items.length === 0 ? 'error' : parsed.warnings.length > 0 ? 'warning' : 'success',
|
||||
error: persistedItems.items.length === 0 ? 'Inga produkter kunde extraheras från flyern.' : null,
|
||||
durationMs: Date.now() - startedAt,
|
||||
});
|
||||
|
||||
return {
|
||||
sessionId: persistedItems.sessionId,
|
||||
retailer: parsed.retailer,
|
||||
@@ -603,10 +628,10 @@ export class FlyerImportService {
|
||||
);
|
||||
|
||||
// 2. Skicka till Mistral Tiny
|
||||
const aiItems = await this.aiParser.parseWithAI(text);
|
||||
const aiParseResult = await this.aiParser.parseWithAI(text);
|
||||
|
||||
// 3. Normalisera resultatet
|
||||
const normalizedItems = this.normalizer.normalize(aiItems);
|
||||
const normalizedItems = this.normalizer.normalize(aiParseResult.items);
|
||||
|
||||
// 4. Konvertera till intern FlyerParseItem-format
|
||||
const items: FlyerParseItem[] = normalizedItems.map((item) => ({
|
||||
@@ -637,6 +662,12 @@ export class FlyerImportService {
|
||||
parserVersion: 'v1',
|
||||
items,
|
||||
warnings,
|
||||
trace: {
|
||||
prompt: aiParseResult.trace.prompt,
|
||||
rawOutput: aiParseResult.trace.rawOutput,
|
||||
chunkCount: aiParseResult.trace.chunkCount,
|
||||
retryCount: aiParseResult.trace.retryCount,
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof BadRequestException) {
|
||||
@@ -652,6 +683,41 @@ export class FlyerImportService {
|
||||
}
|
||||
}
|
||||
|
||||
private async persistFlyerTrace(params: {
|
||||
userId: number;
|
||||
sessionId: number;
|
||||
model: string;
|
||||
prompt: string | null;
|
||||
rawOutput: string | null;
|
||||
normalizedOutput: Record<string, unknown> | null;
|
||||
status: 'success' | 'warning' | 'error';
|
||||
error: string | null;
|
||||
durationMs: number | null;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
await this.prisma.aiTrace.create({
|
||||
data: {
|
||||
source: 'flyer',
|
||||
userId: params.userId,
|
||||
sessionId: params.sessionId,
|
||||
model: params.model,
|
||||
prompt: params.prompt,
|
||||
rawOutput: params.rawOutput,
|
||||
...(params.normalizedOutput == null
|
||||
? {}
|
||||
: { normalizedOutput: params.normalizedOutput as Prisma.InputJsonValue }),
|
||||
status: params.status,
|
||||
error: params.error,
|
||||
durationMs: params.durationMs,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Kunde inte spara flyer AI-trace: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private toFlyerImportItem(item: {
|
||||
id: number;
|
||||
rawName: string;
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { AiFlyerParserService } from './ai-flyer-parser.service';
|
||||
|
||||
describe('AiFlyerParserService dedupe', () => {
|
||||
const service = Object.create(AiFlyerParserService.prototype) as AiFlyerParserService;
|
||||
|
||||
it('dedupes same product with minor offer text differences', () => {
|
||||
const items = [
|
||||
{
|
||||
rawName: 'Kvisttomater',
|
||||
normalizedName: 'kvisttomater',
|
||||
brand: null,
|
||||
category: 'Grönsaker',
|
||||
price: 19.9,
|
||||
priceUnit: 'kg',
|
||||
comparisonPrice: null,
|
||||
comparisonUnit: null,
|
||||
weight: null,
|
||||
bundleWeight: null,
|
||||
isBundle: false,
|
||||
bundleItems: [],
|
||||
offerText: 'Max 2 köp/hushåll',
|
||||
confidence: 0.9,
|
||||
reasonCodes: ['ai_parsed'],
|
||||
},
|
||||
{
|
||||
rawName: 'KVISTTOMATER',
|
||||
normalizedName: 'kvisttomater',
|
||||
brand: null,
|
||||
category: 'Grönsaker',
|
||||
price: 19.9,
|
||||
priceUnit: 'kg',
|
||||
comparisonPrice: null,
|
||||
comparisonUnit: null,
|
||||
weight: null,
|
||||
bundleWeight: null,
|
||||
isBundle: false,
|
||||
bundleItems: [],
|
||||
offerText: 'Max 2 kop/hushall',
|
||||
confidence: 0.89,
|
||||
reasonCodes: ['ai_parsed'],
|
||||
},
|
||||
];
|
||||
|
||||
const result = (service as any).dedupeItems(items);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].normalizedName).toBe('kvisttomater');
|
||||
});
|
||||
|
||||
it('keeps products with same name but different prices', () => {
|
||||
const items = [
|
||||
{
|
||||
rawName: 'Kvisttomater',
|
||||
normalizedName: 'kvisttomater',
|
||||
brand: null,
|
||||
category: 'Grönsaker',
|
||||
price: 19.9,
|
||||
priceUnit: 'kg',
|
||||
comparisonPrice: null,
|
||||
comparisonUnit: null,
|
||||
weight: null,
|
||||
bundleWeight: null,
|
||||
isBundle: false,
|
||||
bundleItems: [],
|
||||
offerText: null,
|
||||
confidence: 0.9,
|
||||
reasonCodes: ['ai_parsed'],
|
||||
},
|
||||
{
|
||||
rawName: 'Kvisttomater',
|
||||
normalizedName: 'kvisttomater',
|
||||
brand: null,
|
||||
category: 'Grönsaker',
|
||||
price: 24.9,
|
||||
priceUnit: 'kg',
|
||||
comparisonPrice: null,
|
||||
comparisonUnit: null,
|
||||
weight: null,
|
||||
bundleWeight: null,
|
||||
isBundle: false,
|
||||
bundleItems: [],
|
||||
offerText: null,
|
||||
confidence: 0.9,
|
||||
reasonCodes: ['ai_parsed'],
|
||||
},
|
||||
];
|
||||
|
||||
const result = (service as any).dedupeItems(items);
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('keeps products with same name/price but materially different campaigns', () => {
|
||||
const items = [
|
||||
{
|
||||
rawName: 'Kvisttomater',
|
||||
normalizedName: 'kvisttomater',
|
||||
brand: null,
|
||||
category: 'Grönsaker',
|
||||
price: 19.9,
|
||||
priceUnit: 'kg',
|
||||
comparisonPrice: null,
|
||||
comparisonUnit: null,
|
||||
weight: null,
|
||||
bundleWeight: null,
|
||||
isBundle: false,
|
||||
bundleItems: [],
|
||||
offerText: 'Max 2 köp/hushåll',
|
||||
confidence: 0.9,
|
||||
reasonCodes: ['ai_parsed'],
|
||||
},
|
||||
{
|
||||
rawName: 'Kvisttomater',
|
||||
normalizedName: 'kvisttomater',
|
||||
brand: null,
|
||||
category: 'Grönsaker',
|
||||
price: 19.9,
|
||||
priceUnit: 'kg',
|
||||
comparisonPrice: null,
|
||||
comparisonUnit: null,
|
||||
weight: null,
|
||||
bundleWeight: null,
|
||||
isBundle: false,
|
||||
bundleItems: [],
|
||||
offerText: 'Ta 3 betala för 2',
|
||||
confidence: 0.9,
|
||||
reasonCodes: ['ai_parsed'],
|
||||
},
|
||||
];
|
||||
|
||||
const result = (service as any).dedupeItems(items);
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('keeps bundle and non-bundle as separate entries', () => {
|
||||
const items = [
|
||||
{
|
||||
rawName: 'Fiskpaket',
|
||||
normalizedName: 'fiskpaket',
|
||||
brand: 'Kapten',
|
||||
category: 'Fisk',
|
||||
price: 49.9,
|
||||
priceUnit: 'pkt',
|
||||
comparisonPrice: 83.17,
|
||||
comparisonUnit: 'kg',
|
||||
weight: null,
|
||||
bundleWeight: '600g',
|
||||
isBundle: true,
|
||||
bundleItems: ['A', 'B'],
|
||||
offerText: null,
|
||||
confidence: 0.9,
|
||||
reasonCodes: ['ai_parsed'],
|
||||
},
|
||||
{
|
||||
rawName: 'Fiskpaket',
|
||||
normalizedName: 'fiskpaket',
|
||||
brand: 'Kapten',
|
||||
category: 'Fisk',
|
||||
price: 49.9,
|
||||
priceUnit: 'pkt',
|
||||
comparisonPrice: 83.17,
|
||||
comparisonUnit: 'kg',
|
||||
weight: null,
|
||||
bundleWeight: null,
|
||||
isBundle: false,
|
||||
bundleItems: [],
|
||||
offerText: null,
|
||||
confidence: 0.9,
|
||||
reasonCodes: ['ai_parsed'],
|
||||
},
|
||||
];
|
||||
|
||||
const result = (service as any).dedupeItems(items);
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('throws for empty input in parseWithAI', async () => {
|
||||
await expect((service as any).parseWithAI('')).rejects.toBeInstanceOf(
|
||||
BadRequestException,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -25,6 +25,13 @@ export interface AiFlyerParseResult {
|
||||
reasonCodes: string[];
|
||||
}
|
||||
|
||||
export interface AiFlyerParseTrace {
|
||||
prompt: string | null;
|
||||
rawOutput: string | null;
|
||||
chunkCount: number;
|
||||
retryCount: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AiFlyerParserService {
|
||||
private readonly logger = new Logger(AiFlyerParserService.name);
|
||||
@@ -66,7 +73,7 @@ export class AiFlyerParserService {
|
||||
* @param text Text från flyern (från pdf-parse eller OCR)
|
||||
* @returns Array av parsade produkter
|
||||
*/
|
||||
async parseWithAI(text: string): Promise<AiFlyerParseResult[]> {
|
||||
async parseWithAI(text: string): Promise<{ items: AiFlyerParseResult[]; trace: AiFlyerParseTrace }> {
|
||||
if (!text || text.trim().length === 0) {
|
||||
throw new BadRequestException('Flyer-texten är tom. Kan inte fortsätta.');
|
||||
}
|
||||
@@ -95,18 +102,30 @@ export class AiFlyerParserService {
|
||||
}
|
||||
|
||||
const allItems: AiFlyerParseResult[] = [];
|
||||
const prompts: string[] = [];
|
||||
const rawResponses: string[] = [];
|
||||
let retryCount = 0;
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunkItems = await this.parseChunkWithRetry(
|
||||
const chunkResult = await this.parseChunkWithRetry(
|
||||
client,
|
||||
chunks[i],
|
||||
i + 1,
|
||||
chunks.length,
|
||||
debugSession,
|
||||
);
|
||||
allItems.push(...chunkItems);
|
||||
allItems.push(...chunkResult.items);
|
||||
prompts.push(chunkResult.prompt);
|
||||
rawResponses.push(chunkResult.rawOutput);
|
||||
retryCount += Math.max(0, chunkResult.attemptsUsed - 1);
|
||||
}
|
||||
|
||||
const deduped = this.dedupeItems(allItems);
|
||||
const trace: AiFlyerParseTrace = {
|
||||
prompt: prompts.length > 0 ? prompts.join('\n\n-----\n\n') : null,
|
||||
rawOutput: rawResponses.length > 0 ? rawResponses.join('\n\n-----\n\n') : null,
|
||||
chunkCount: chunks.length,
|
||||
retryCount,
|
||||
};
|
||||
|
||||
if (debugSession) {
|
||||
await this.writeDebugFile(
|
||||
@@ -116,7 +135,7 @@ export class AiFlyerParserService {
|
||||
);
|
||||
}
|
||||
|
||||
return deduped;
|
||||
return { items: deduped, trace };
|
||||
} catch (err) {
|
||||
if (debugSession) {
|
||||
await this.writeDebugFile(
|
||||
@@ -371,7 +390,12 @@ ${truncatedText}`;
|
||||
chunkIndex: number,
|
||||
totalChunks: number,
|
||||
debugSession: { dirPath: string; baseName: string } | null,
|
||||
): Promise<AiFlyerParseResult[]> {
|
||||
): Promise<{
|
||||
items: AiFlyerParseResult[];
|
||||
prompt: string;
|
||||
rawOutput: string;
|
||||
attemptsUsed: number;
|
||||
}> {
|
||||
const textWindows = [3000, 2200, 1600];
|
||||
const attempts = Math.max(1, Math.min(this.maxRetries + 1, textWindows.length));
|
||||
let lastError: unknown = null;
|
||||
@@ -425,7 +449,12 @@ ${truncatedText}`;
|
||||
throw new BadRequestException('AI returnerade inte en JSON-array.');
|
||||
}
|
||||
|
||||
return items.map((aiItem, idx) => this.normalizeAiItem(aiItem, idx));
|
||||
return {
|
||||
items: items.map((aiItem, idx) => this.normalizeAiItem(aiItem, idx)),
|
||||
prompt,
|
||||
rawOutput: String(content),
|
||||
attemptsUsed: i + 1,
|
||||
};
|
||||
} catch (attemptErr) {
|
||||
lastError = attemptErr;
|
||||
if (debugSession) {
|
||||
@@ -454,14 +483,24 @@ ${truncatedText}`;
|
||||
const deduped: AiFlyerParseResult[] = [];
|
||||
|
||||
for (const item of items) {
|
||||
const normalizedName = item.normalizedName.trim();
|
||||
const normalizedBrand = (item.brand ?? '').trim().toLowerCase();
|
||||
const normalizedPrice = item.price == null ? '' : Number(item.price).toFixed(2);
|
||||
const normalizedPriceUnit = (item.priceUnit ?? '').trim().toLowerCase();
|
||||
const normalizedComparisonPrice =
|
||||
item.comparisonPrice == null ? '' : Number(item.comparisonPrice).toFixed(2);
|
||||
const normalizedComparisonUnit = (item.comparisonUnit ?? '').trim().toLowerCase();
|
||||
const offerSignature = this.offerSignature(item.offerText);
|
||||
|
||||
const key = [
|
||||
item.normalizedName,
|
||||
item.price ?? '',
|
||||
item.priceUnit ?? '',
|
||||
item.offerText ?? '',
|
||||
normalizedName,
|
||||
normalizedBrand,
|
||||
normalizedPrice,
|
||||
normalizedPriceUnit,
|
||||
normalizedComparisonPrice,
|
||||
normalizedComparisonUnit,
|
||||
offerSignature,
|
||||
item.isBundle ? '1' : '0',
|
||||
item.bundleWeight ?? '',
|
||||
JSON.stringify(item.bundleItems ?? []),
|
||||
].join('|');
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
@@ -471,6 +510,27 @@ ${truncatedText}`;
|
||||
return deduped;
|
||||
}
|
||||
|
||||
private offerSignature(offerText: string | null | undefined): string {
|
||||
if (!offerText || offerText.trim().length === 0) return '';
|
||||
|
||||
const normalized = offerText
|
||||
.toLowerCase()
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.replace(/[^a-z0-9\s]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
if (!normalized) return '';
|
||||
|
||||
const hasCampaignMarkers =
|
||||
/(max|hogst|begransat|hushall|kund|kop|for|betala|ta)/.test(normalized)
|
||||
|| /(\d+\s*for\s*\d+)/.test(normalized)
|
||||
|| /(ta\s*\d+\s*betala\s*for\s*\d+)/.test(normalized);
|
||||
|
||||
return hasCampaignMarkers ? normalized : '';
|
||||
}
|
||||
|
||||
private readPositiveIntEnv(key: string, fallback: number): number {
|
||||
const raw = process.env[key];
|
||||
if (!raw) return fallback;
|
||||
|
||||
@@ -120,7 +120,7 @@ describe('FlyerNormalizerService', () => {
|
||||
const result = service.normalize(items);
|
||||
|
||||
expect(result).toHaveLength(3);
|
||||
expect(result.map((item) => item.rawName)).toEqual(['Prastost', 'Herrgardsost', 'Greveost']);
|
||||
expect(result.map((item) => item.rawName)).toEqual(['Prästost', 'Herrgårdsost', 'Greveost']);
|
||||
expect(result.every((item) => item.brand === 'Arla Ko')).toBe(true);
|
||||
expect(result.every((item) => item.categoryHint === 'Hårdost')).toBe(true);
|
||||
expect(result[0].parseReasons).toContain('split_cheese_variants');
|
||||
@@ -141,5 +141,48 @@ describe('FlyerNormalizerService', () => {
|
||||
expect(result[0].brand).toBe('Arla Ko');
|
||||
expect(result[0].categoryHint).toBe('Hårdost');
|
||||
});
|
||||
|
||||
it('fixes known OCR typo for spröd', () => {
|
||||
const items = [
|
||||
{
|
||||
rawName: 'Pröd Bakad Firre',
|
||||
brand: 'Findus',
|
||||
},
|
||||
];
|
||||
|
||||
const result = service.normalize(items);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].rawName).toBe('Spröd Bakad Firre');
|
||||
expect(result[0].normalizedName).toBe('spröd bakad firre');
|
||||
});
|
||||
|
||||
it('does not apply spröd typo fix outside known fish context', () => {
|
||||
const items = [
|
||||
{
|
||||
rawName: 'Pröd tvättmedel',
|
||||
brand: 'Test',
|
||||
},
|
||||
];
|
||||
|
||||
const result = service.normalize(items);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].rawName).toBe('Pröd tvättmedel');
|
||||
});
|
||||
|
||||
it('fixes herggårdsost only in cheese context', () => {
|
||||
const items = [
|
||||
{
|
||||
rawName: 'Herggårdsost 31%',
|
||||
brand: 'Arla Ko',
|
||||
},
|
||||
];
|
||||
|
||||
const result = service.normalize(items);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].rawName).toContain('Herrgårdsost');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -24,8 +24,8 @@ export class FlyerNormalizerService {
|
||||
private readonly MAX_BUNDLE_ITEMS = 20;
|
||||
private readonly MAX_BUNDLE_ITEM_LENGTH = 120;
|
||||
private readonly CHEESE_VARIANT_TO_NAME: Record<string, string> = {
|
||||
prast: 'Prastost',
|
||||
herrgard: 'Herrgardsost',
|
||||
prast: 'Prästost',
|
||||
herrgard: 'Herrgårdsost',
|
||||
greve: 'Greveost',
|
||||
};
|
||||
|
||||
@@ -76,11 +76,12 @@ export class FlyerNormalizerService {
|
||||
return [null];
|
||||
}
|
||||
|
||||
const rawName = this.extractString(item.rawName) || this.extractString(item.name);
|
||||
if (!rawName) {
|
||||
const rawNameValue = this.extractString(item.rawName) || this.extractString(item.name);
|
||||
if (!rawNameValue) {
|
||||
this.logger.warn(`Item ${index} has no name, skipping`);
|
||||
return [null];
|
||||
}
|
||||
const rawName = this.fixKnownOcrTypos(rawNameValue);
|
||||
|
||||
const normalizedName = this.extractString(item.normalizedName) || this.normalizeName(rawName);
|
||||
const normalizedBrand = this.normalizeBrand(this.extractString(item.brand), rawName);
|
||||
@@ -245,4 +246,18 @@ export class FlyerNormalizerService {
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private fixKnownOcrTypos(value: string): string {
|
||||
let corrected = value;
|
||||
|
||||
if (/\bbakad\b/i.test(value) && /\bfirre\b/i.test(value)) {
|
||||
corrected = corrected.replace(/\bpröd\b/gi, (match) => (match[0] === 'P' ? 'Spröd' : 'spröd'));
|
||||
}
|
||||
|
||||
if (/ost\b|hårdost/i.test(value)) {
|
||||
corrected = corrected.replace(/\bherg{1,2}årds?ost\b/gi, (match) => (match[0] === 'H' ? 'Herrgårdsost' : 'herrgårdsost'));
|
||||
}
|
||||
|
||||
return corrected;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,10 +300,13 @@ class _AdminAiPanelState extends ConsumerState<AdminAiPanel> {
|
||||
title: Text(item.fileName ?? item.id),
|
||||
subtitle: Text(
|
||||
'${_formatDateTime(item.createdAt)} • ${item.userLabel}'),
|
||||
trailing: Chip(
|
||||
label: Text(item.status.label),
|
||||
labelStyle: TextStyle(
|
||||
color: _statusColor(item.status, theme.colorScheme)),
|
||||
trailing: Tooltip(
|
||||
message: _statusTooltipText(item),
|
||||
child: Chip(
|
||||
label: Text(item.status.label),
|
||||
labelStyle: TextStyle(
|
||||
color: _statusColor(item.status, theme.colorScheme)),
|
||||
),
|
||||
),
|
||||
);
|
||||
},
|
||||
@@ -349,6 +352,14 @@ class _AdminAiPanelState extends ConsumerState<AdminAiPanel> {
|
||||
return ListView(
|
||||
children: [
|
||||
_TraceMetaCard(detail: detail, formatDateTime: _formatDateTime),
|
||||
if (detail.warnings.isNotEmpty) ...[
|
||||
const SizedBox(height: 12),
|
||||
_WarningsCard(
|
||||
warnings: detail.warnings,
|
||||
onCopyWarning: (warning) => _copyText(warning, 'Varning'),
|
||||
onCopyAll: () => _copyText(detail.warnings.join('\n'), 'Varningar'),
|
||||
),
|
||||
],
|
||||
const SizedBox(height: 12),
|
||||
_PromptCard(
|
||||
prompt: prompt,
|
||||
@@ -375,6 +386,20 @@ class _AdminAiPanelState extends ConsumerState<AdminAiPanel> {
|
||||
_cachedOutputPrettyJson = next;
|
||||
return next;
|
||||
}
|
||||
|
||||
String _statusTooltipText(AdminAiTraceListItem item) {
|
||||
final parts = <String>[];
|
||||
if (item.status == AdminAiTraceStatus.warning && item.warningsCount > 0) {
|
||||
parts.add('${item.warningsCount} varning(ar). Välj raden för detaljer och kopiering.');
|
||||
}
|
||||
if (item.error != null && item.error!.trim().isNotEmpty) {
|
||||
parts.add(item.error!.trim());
|
||||
}
|
||||
if (parts.isEmpty) {
|
||||
return 'Inga ytterligare detaljer.';
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
}
|
||||
|
||||
class _TraceMetaCard extends StatelessWidget {
|
||||
@@ -466,15 +491,13 @@ class _PromptCard extends StatelessWidget {
|
||||
.withValues(alpha: 0.35),
|
||||
borderRadius: BorderRadius.circular(8),
|
||||
),
|
||||
child: Text(
|
||||
hasPrompt
|
||||
? value
|
||||
: 'Prompt är inte tillgänglig i denna fas för vald källa.',
|
||||
maxLines: expanded ? null : 10,
|
||||
overflow:
|
||||
expanded ? TextOverflow.visible : TextOverflow.ellipsis,
|
||||
style: theme.textTheme.bodySmall
|
||||
?.copyWith(fontFamily: 'monospace'),
|
||||
child: SelectionArea(
|
||||
child: SelectableText(
|
||||
hasPrompt ? value : 'Prompt saknas för detta spår.',
|
||||
maxLines: expanded ? null : 10,
|
||||
style: theme.textTheme.bodySmall
|
||||
?.copyWith(fontFamily: 'monospace'),
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -484,15 +507,27 @@ class _PromptCard extends StatelessWidget {
|
||||
}
|
||||
}
|
||||
|
||||
class _OutputJsonCard extends StatelessWidget {
|
||||
class _OutputJsonCard extends StatefulWidget {
|
||||
final String jsonText;
|
||||
final VoidCallback onCopy;
|
||||
|
||||
const _OutputJsonCard({required this.jsonText, required this.onCopy});
|
||||
|
||||
@override
|
||||
State<_OutputJsonCard> createState() => _OutputJsonCardState();
|
||||
}
|
||||
|
||||
class _OutputJsonCardState extends State<_OutputJsonCard> {
|
||||
static const int _maxPreviewChars = 12000;
|
||||
bool _expanded = false;
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final theme = Theme.of(context);
|
||||
final shouldTruncate = widget.jsonText.length > _maxPreviewChars;
|
||||
final visibleText =
|
||||
!_expanded && shouldTruncate ? '${widget.jsonText.substring(0, _maxPreviewChars)}\n…' : widget.jsonText;
|
||||
|
||||
return Card(
|
||||
child: Padding(
|
||||
padding: const EdgeInsets.all(16),
|
||||
@@ -506,11 +541,20 @@ class _OutputJsonCard extends StatelessWidget {
|
||||
style: theme.textTheme.titleMedium)),
|
||||
IconButton(
|
||||
tooltip: 'Kopiera JSON',
|
||||
onPressed: onCopy,
|
||||
onPressed: widget.onCopy,
|
||||
icon: const Icon(Icons.copy_all),
|
||||
),
|
||||
],
|
||||
),
|
||||
if (shouldTruncate)
|
||||
Align(
|
||||
alignment: Alignment.centerRight,
|
||||
child: TextButton.icon(
|
||||
onPressed: () => setState(() => _expanded = !_expanded),
|
||||
icon: Icon(_expanded ? Icons.unfold_less : Icons.unfold_more),
|
||||
label: Text(_expanded ? 'Visa mindre' : 'Visa hela outputen'),
|
||||
),
|
||||
),
|
||||
const SizedBox(height: 8),
|
||||
Container(
|
||||
width: double.infinity,
|
||||
@@ -520,12 +564,70 @@ class _OutputJsonCard extends StatelessWidget {
|
||||
.withValues(alpha: 0.35),
|
||||
borderRadius: BorderRadius.circular(8),
|
||||
),
|
||||
child: SingleChildScrollView(
|
||||
scrollDirection: Axis.horizontal,
|
||||
child: Text(
|
||||
jsonText,
|
||||
style: theme.textTheme.bodySmall
|
||||
?.copyWith(fontFamily: 'monospace'),
|
||||
child: SelectionArea(
|
||||
child: SingleChildScrollView(
|
||||
scrollDirection: Axis.horizontal,
|
||||
child: SelectableText(
|
||||
visibleText,
|
||||
style: theme.textTheme.bodySmall
|
||||
?.copyWith(fontFamily: 'monospace'),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class _WarningsCard extends StatelessWidget {
|
||||
final List<String> warnings;
|
||||
final void Function(String warning) onCopyWarning;
|
||||
final VoidCallback onCopyAll;
|
||||
|
||||
const _WarningsCard({
|
||||
required this.warnings,
|
||||
required this.onCopyWarning,
|
||||
required this.onCopyAll,
|
||||
});
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final theme = Theme.of(context);
|
||||
return Card(
|
||||
child: Padding(
|
||||
padding: const EdgeInsets.all(16),
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
Row(
|
||||
children: [
|
||||
Expanded(
|
||||
child: Text(
|
||||
'Varningar (${warnings.length})',
|
||||
style: theme.textTheme.titleMedium,
|
||||
),
|
||||
),
|
||||
IconButton(
|
||||
tooltip: 'Kopiera alla varningar',
|
||||
onPressed: onCopyAll,
|
||||
icon: const Icon(Icons.copy_all),
|
||||
),
|
||||
],
|
||||
),
|
||||
const SizedBox(height: 8),
|
||||
...warnings.map(
|
||||
(warning) => ListTile(
|
||||
dense: true,
|
||||
contentPadding: EdgeInsets.zero,
|
||||
leading: const Icon(Icons.warning_amber_rounded, size: 18),
|
||||
title: SelectableText(warning),
|
||||
trailing: IconButton(
|
||||
tooltip: 'Kopiera varning',
|
||||
onPressed: () => onCopyWarning(warning),
|
||||
icon: const Icon(Icons.copy, size: 18),
|
||||
),
|
||||
),
|
||||
),
|
||||
|
||||
@@ -54,6 +54,8 @@ Widget _buildPanelApp(AdminRepository repo) {
|
||||
}
|
||||
|
||||
void main() {
|
||||
final veryLargeOutput = '{"payload":"${List.filled(13050, 'x').join()}"}';
|
||||
|
||||
final flyerItem = AdminAiTraceListItem(
|
||||
id: 'flyer-101',
|
||||
source: AdminAiTraceSource.flyer,
|
||||
@@ -68,7 +70,7 @@ void main() {
|
||||
warningsCount: 2,
|
||||
hasPrompt: true,
|
||||
hasOutput: true,
|
||||
error: null,
|
||||
error: 'Det finns 2 varningar i detaljvyn.',
|
||||
);
|
||||
|
||||
final flyerDetail = AdminAiTraceDetail(
|
||||
@@ -84,16 +86,11 @@ void main() {
|
||||
durationMs: 1880,
|
||||
retryCount: 1,
|
||||
chunkCount: 3,
|
||||
warnings: const ['parse:low_confidence'],
|
||||
warnings: const ['parse:low_confidence', 'match:no_match'],
|
||||
error: null,
|
||||
prompt: 'Prompttext exempel',
|
||||
rawOutput: '{"ok":true}',
|
||||
normalizedOutput: const {
|
||||
'sessionId': 101,
|
||||
'items': [
|
||||
{'rawName': 'Tomat'}
|
||||
],
|
||||
},
|
||||
rawOutput: veryLargeOutput,
|
||||
normalizedOutput: null,
|
||||
summary: const {'itemCount': 1},
|
||||
);
|
||||
|
||||
@@ -180,7 +177,7 @@ void main() {
|
||||
expect(find.text('willys-v20.pdf'), findsOneWidget);
|
||||
});
|
||||
|
||||
testWidgets('Prompt and output render and copy actions show snackbars',
|
||||
testWidgets('Prompt/output are selectable and warning details are visible',
|
||||
(tester) async {
|
||||
await tester.binding.setSurfaceSize(const Size(1400, 1200));
|
||||
final fakeRepo = _FakeAdminRepository(
|
||||
@@ -200,6 +197,9 @@ void main() {
|
||||
await tester.pump(const Duration(milliseconds: 500));
|
||||
|
||||
expect(find.text('Sammanfattning'), findsOneWidget);
|
||||
expect(find.text('Varningar (2)'), findsOneWidget);
|
||||
expect(find.text('parse:low_confidence'), findsOneWidget);
|
||||
expect(find.text('match:no_match'), findsOneWidget);
|
||||
final detailScroll = find.byType(Scrollable).last;
|
||||
await tester.scrollUntilVisible(
|
||||
find.text('Model Output'),
|
||||
@@ -209,12 +209,19 @@ void main() {
|
||||
await tester.pumpAndSettle();
|
||||
|
||||
expect(find.text('Model Output'), findsOneWidget);
|
||||
expect(find.textContaining('"sessionId": 101'), findsOneWidget);
|
||||
expect(find.byType(SelectableText), findsWidgets);
|
||||
expect(find.text('Visa hela outputen'), findsOneWidget);
|
||||
|
||||
await tester.tap(find.text('Visa hela outputen'));
|
||||
await tester.pumpAndSettle();
|
||||
expect(find.text('Visa mindre'), findsOneWidget);
|
||||
|
||||
final copyPrompt = find.byTooltip('Kopiera');
|
||||
final copyOutput = find.byTooltip('Kopiera JSON');
|
||||
final copyWarnings = find.byTooltip('Kopiera alla varningar');
|
||||
expect(copyPrompt, findsOneWidget);
|
||||
expect(copyOutput, findsOneWidget);
|
||||
expect(copyWarnings, findsOneWidget);
|
||||
|
||||
await tester.tap(copyPrompt);
|
||||
await tester.pumpAndSettle();
|
||||
@@ -224,6 +231,10 @@ void main() {
|
||||
await tester.pumpAndSettle();
|
||||
expect(tester.takeException(), isNull);
|
||||
|
||||
await tester.tap(copyWarnings);
|
||||
await tester.pumpAndSettle();
|
||||
expect(tester.takeException(), isNull);
|
||||
|
||||
addTearDown(() => tester.binding.setSurfaceSize(null));
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user