feat(flyer-import): add detailed product signals and display names

- Added `signals` and `displayNameDetailed` fields to FlyerItem model in Prisma schema - Introduced `FlyerImportSignals` type with origin countries, labels, quality flags, variant, and packaging - Added `displayNameDetailed` field to FlyerImportItem DTO and Flutter model - Implemented utility functions for signal extraction and display name building - Updated flyer import service to persist and return signals/category data - Enhanced Flutter UI to display detailed product information including badges for signals - Added new test coverage for signals persistence and display name generation - Added new import-common module for shared import utilities - Created database migration for new fields - Added Kilo plan for feature development
2026-05-24 19:32:13 +02:00
parent d9f992ca9a
commit b04d157915
16 changed files with 1124 additions and 107 deletions
@@ -6,9 +6,10 @@ import {
  NotFoundException,
  ServiceUnavailableException,
 } from '@nestjs/common';
-import { Prisma } from '@prisma/client';
-import { PrismaService } from '../prisma/prisma.service';
-import { normalizeName } from '../common/utils/normalize-name';
+import { Prisma } from '@prisma/client';
+import { PrismaService } from '../prisma/prisma.service';
+import { normalizeName } from '../common/utils/normalize-name';
+import { CategoriesService } from '../categories/categories.service';
 import {
  FlyerImportItem,
  FlyerImportMatchVia,
@@ -18,6 +19,10 @@ import { TextExtractorService } from './services/text-extractor.service';
 import { AiFlyerParserService } from './services/ai-flyer-parser.service';
 import { FlyerNormalizerService } from './services/flyer-normalizer.service';
 import { describeMatchReason, describeParseReason } from './services/reason-codes';
+import { CategoryResolverService } from '../import-common/category-resolver.service';
+import { buildDisplayNameDetailed } from '../import-common/import-display-name.util';
+import { extractImportSignals } from '../import-common/import-signals.util';
+import { ImportedItemSignals } from '../import-common/import-item.types';

 type FlyerParseItem = {
  rawName: string;
@@ -58,11 +63,12 @@ type ExtractedOfferSignals = {
  hasCampaignPattern: boolean;
 };

-type ProductLite = {
-  id: number;
-  name: string;
-  canonicalName: string | null;
-};
+type ProductLite = {
+  id: number;
+  name: string;
+  canonicalName: string | null;
+  categoryId: number | null;
+};

@Injectable()
 export class FlyerImportService {
@@ -70,29 +76,37 @@ export class FlyerImportService {
  private readonly MAX_BUNDLE_ITEMS = 20;
  private readonly MAX_BUNDLE_ITEM_LENGTH = 120;

-  constructor(
-    private readonly prisma: PrismaService,
-    private readonly textExtractor: TextExtractorService,
-    private readonly aiParser: AiFlyerParserService,
-    private readonly normalizer: FlyerNormalizerService,
-  ) {}
+  constructor(
+    private readonly prisma: PrismaService,
+    private readonly categoriesService: CategoriesService,
+    private readonly categoryResolver: CategoryResolverService,
+    private readonly textExtractor: TextExtractorService,
+    private readonly aiParser: AiFlyerParserService,
+    private readonly normalizer: FlyerNormalizerService,
+  ) {}

  async parseAndMatch(file: Express.Multer.File, userId: number): Promise<FlyerImportResponse> {
    const startedAt = Date.now();
    const parsed = await this.parseViaInternal(file);

-    const [products, aliases] = await Promise.all([
-      this.prisma.product.findMany({
-        where: { ownerId: userId, isActive: true },
-        select: { id: true, name: true, canonicalName: true },
-      }),
-      this.prisma.receiptAlias.findMany({
-        where: {
-          OR: [{ ownerId: userId, isGlobal: false }, { isGlobal: true }],
-        },
-        select: { receiptName: true, productId: true },
-      }),
-    ]);
+    const [products, aliases, categories] = await Promise.all([
+      this.prisma.product.findMany({
+        where: { ownerId: userId, isActive: true },
+        select: { id: true, name: true, canonicalName: true, categoryId: true },
+      }),
+      this.prisma.receiptAlias.findMany({
+        where: {
+          OR: [{ ownerId: userId, isGlobal: false }, { isGlobal: true }],
+        },
+        select: { receiptName: true, productId: true },
+      }),
+      this.categoriesService.findFlattened().catch((error) => {
+        this.logger.warn(
+          `Could not load categories for flyer import, proceeding without rule categories: ${error instanceof Error ? error.message : String(error)}`,
+        );
+        return [];
+      }),
+    ]);

    const aliasToProduct = new Map<string, number>();
    for (const alias of aliases) {
@@ -109,20 +123,39 @@ export class FlyerImportService {
    }

    const items: FlyerImportItem[] = parsed.items.map((item) => {
-      const match = this.matchItem(item, products, aliasToProduct, productById);
+      const signalData = extractImportSignals({
+        rawName: item.rawName,
+        brand: item.brand,
+        offerText: item.offerText,
+      });
+
+      const match = this.matchItem(item, signalData.normalizedMatchName, signalData.signals, products, aliasToProduct, productById);
      const signals = this.extractOfferSignals(item.offerText);
      const price = item.price ?? signals.price;
      const priceUnit = this.normalizeUnit(item.priceUnit) ?? signals.priceUnit;
      const comparisonPrice = item.comparisonPrice ?? signals.comparisonPrice;
      const comparisonUnit = this.normalizeUnit(item.comparisonUnit) ?? signals.comparisonUnit;
      const offerLimitText = this.extractOfferLimitText(item.offerText);
+      const displayNameDetailed = buildDisplayNameDetailed({
+        rawName: item.rawName,
+        isBundle: item.isBundle,
+        bundleItems: this.sanitizeBundleItems(item.bundleItems),
+      });
+      const categoryId = this.categoryResolver.resolveForFlyer({
+        categories,
+        signalText: [item.rawName, item.brand ?? '', item.offerText ?? ''].join(' ').trim(),
+        categoryHint: item.category,
+        matchedProductCategoryId: match.product?.categoryId ?? null,
+        matchConfidence: match.confidence,
+      });
+
      return {
        flyerItemId: null,
        rawName: item.rawName,
-        normalizedName: item.normalizedName,
+        normalizedName: signalData.normalizedMatchName || item.normalizedName,
        brand: item.brand,
        category: item.category,
-        categoryId: null,
+        categoryId,
        price,
        priceUnit,
        comparisonPrice,
@@ -131,6 +164,8 @@ export class FlyerImportService {
        bundleWeight: item.bundleWeight,
        isBundle: item.isBundle,
        bundleItems: this.sanitizeBundleItems(item.bundleItems),
+        displayNameDetailed,
+        signals: signalData.signals,
        offerText: item.offerText,
        isOffer: this.isOfferItem(item, signals.hasCampaignPattern),
        offerLimitText,
@@ -145,6 +180,8 @@ export class FlyerImportService {
        matchReasonsDetailed: this.describeMatchReasons(match.reasons),
      };
    });
+
+    this.logImportMetrics(items);

    const persistedItems = await this.persistSessionWithItems(userId, parsed.retailer, items, file);

@@ -385,36 +422,40 @@ export class FlyerImportService {
      select: { id: true },
    });

-    const savedItems: FlyerImportItem[] = [];
-    for (const item of items) {
-      const created = await this.prisma.flyerItem.create({
-        data: {
-          sessionId: session.id,
-          rawName: item.rawName,
-          normalizedName: item.normalizedName,
-          brand: item.brand,
-          categoryHint: item.category,
-          categoryId: item.categoryId,
-          price: item.price != null ? new Prisma.Decimal(item.price) : null,
-          priceUnit: item.priceUnit,
-          comparisonPrice:
-            item.comparisonPrice != null ? new Prisma.Decimal(item.comparisonPrice) : null,
-          comparisonUnit: item.comparisonUnit,
-          weight: item.weight,
-          bundleWeight: item.bundleWeight,
-          isBundle: item.isBundle,
-          bundleItems: item.bundleItems,
-          offerText: item.offerText,
-          parseConfidence: item.parseConfidence,
-          parseReasons: item.parseReasons,
-          matchedProductId: item.matchedProductId,
-          matchedProductName: item.matchedProductName,
-          matchedVia: item.matchedVia,
-          matchConfidence: item.matchConfidence,
-          matchReasons: item.matchReasons,
-        },
-        select: { id: true },
-      });
+    const savedItems: FlyerImportItem[] = [];
+    for (const item of items) {
+      const createData: Prisma.FlyerItemUncheckedCreateInput = {
+        sessionId: session.id,
+        rawName: item.rawName,
+        normalizedName: item.normalizedName,
+        brand: item.brand,
+        categoryHint: item.category,
+        categoryId: item.categoryId,
+        price: item.price != null ? new Prisma.Decimal(item.price) : null,
+        priceUnit: item.priceUnit,
+        comparisonPrice:
+          item.comparisonPrice != null ? new Prisma.Decimal(item.comparisonPrice) : null,
+        comparisonUnit: item.comparisonUnit,
+        weight: item.weight,
+        bundleWeight: item.bundleWeight,
+        isBundle: item.isBundle,
+        bundleItems: item.bundleItems,
+        displayNameDetailed: item.displayNameDetailed,
+        signals: item.signals as Prisma.InputJsonValue,
+        offerText: item.offerText,
+        parseConfidence: item.parseConfidence,
+        parseReasons: item.parseReasons,
+        matchedProductId: item.matchedProductId,
+        matchedProductName: item.matchedProductName,
+        matchedVia: item.matchedVia,
+        matchConfidence: item.matchConfidence,
+        matchReasons: item.matchReasons,
+      };
+
+      const created = await this.prisma.flyerItem.create({
+        data: createData,
+        select: { id: true },
+      });

      savedItems.push({ ...item, flyerItemId: created.id });
    }
@@ -431,21 +472,23 @@ export class FlyerImportService {
    return `${d.getUTCFullYear()}-W${String(weekNo).padStart(2, '0')}`;
  }

-  private matchItem(
-    item: FlyerParseItem,
-    products: ProductLite[],
-    aliasToProduct: Map<string, number>,
-    productById: Map<number, ProductLite>,
-  ): {
+  private matchItem(
+    item: FlyerParseItem,
+    normalizedMatchName: string,
+    itemSignals: ImportedItemSignals,
+    products: ProductLite[],
+    aliasToProduct: Map<string, number>,
+    productById: Map<number, ProductLite>,
+  ): {
    product: ProductLite | null;
    via: FlyerImportMatchVia;
    confidence: number;
    reasons: string[];
  } {
-    const normalized = normalizeName(item.rawName || item.normalizedName);
-    if (!normalized) {
-      return { product: null, via: 'none', confidence: 0, reasons: ['empty_name'] };
-    }
+    const normalized = normalizedMatchName || normalizeName(item.normalizedName || item.rawName);
+    if (!normalized) {
+      return { product: null, via: 'none', confidence: 0, reasons: ['empty_name'] };
+    }

    const aliasedProductId = aliasToProduct.get(normalized);
    if (aliasedProductId) {
@@ -471,17 +514,30 @@ export class FlyerImportService {
      }
    }

-    let best: { product: ProductLite; confidence: number; overlap: number } | null = null;
-    const itemTokens = this.tokenize(item.rawName);
-    for (const product of products) {
-      const productTokens = this.tokenize(product.canonicalName ?? product.name);
-      const overlap = this.tokenOverlap(itemTokens, productTokens);
-      if (overlap <= 0) continue;
-      const confidence = Math.min(0.92, 0.5 + overlap * 0.4);
-      if (!best || confidence > best.confidence) {
-        best = { product, confidence, overlap };
-      }
-    }
+    let best: { product: ProductLite; confidence: number; overlap: number } | null = null;
+    const itemTokens = this.tokenize(normalized);
+    for (const product of products) {
+      const productTokens = this.tokenize(product.canonicalName ?? product.name);
+      const overlap = this.tokenOverlap(itemTokens, productTokens);
+      if (overlap <= 0) continue;
+
+      let confidence = Math.min(0.93, 0.48 + overlap * 0.42);
+      if (this.hasBrandSignal(item.brand, product)) {
+        confidence += 0.04;
+      }
+      if (this.hasWeightSignal(item.weight, product)) {
+        confidence += 0.03;
+      }
+      if (this.hasQualitySignal(itemSignals, product)) {
+        confidence += 0.03;
+      }
+
+      confidence = Math.min(0.95, confidence);
+
+      if (!best || confidence > best.confidence) {
+        best = { product, confidence, overlap };
+      }
+    }

    if (best && best.confidence >= 0.66) {
      return {
@@ -508,7 +564,7 @@ export class FlyerImportService {
      .filter((part) => part.length >= 3);
  }

-  private tokenOverlap(a: string[], b: string[]): number {
+  private tokenOverlap(a: string[], b: string[]): number {
    if (a.length === 0 || b.length === 0) return 0;
    const as = new Set(a);
    const bs = new Set(b);
@@ -518,8 +574,32 @@ export class FlyerImportService {
    }
    const union = new Set([...as, ...bs]).size;
    if (union === 0) return 0;
-    return intersection / union;
-  }
+    return intersection / union;
+  }
+
+  private hasBrandSignal(brand: string | null, product: ProductLite): boolean {
+    if (!brand) return false;
+    const normalizedBrand = normalizeName(brand);
+    if (!normalizedBrand) return false;
+
+    const normalizedProduct = normalizeName(`${product.name} ${product.canonicalName ?? ''}`);
+    return normalizedProduct.includes(normalizedBrand);
+  }
+
+  private hasWeightSignal(weight: string | null, product: ProductLite): boolean {
+    if (!weight) return false;
+    const normalizedWeight = normalizeName(weight);
+    if (!normalizedWeight) return false;
+
+    const normalizedProduct = normalizeName(`${product.name} ${product.canonicalName ?? ''}`);
+    return normalizedProduct.includes(normalizedWeight);
+  }
+
+  private hasQualitySignal(signals: ImportedItemSignals, product: ProductLite): boolean {
+    if (!signals.qualityFlags.includes('eco')) return false;
+    const normalizedProduct = normalizeName(`${product.name} ${product.canonicalName ?? ''}`);
+    return /\beko\b|\bekolog/i.test(normalizedProduct);
+  }

  private isOfferItem(item: FlyerParseItem, hasCampaignPattern: boolean): boolean {
    return (
@@ -745,6 +825,8 @@ export class FlyerImportService {
    bundleWeight: string | null;
    isBundle: boolean;
    bundleItems: Prisma.JsonValue | null;
+    displayNameDetailed?: string | null;
+    signals?: Prisma.JsonValue | null;
    offerText: string | null;
    parseConfidence: number;
    parseReasons: Prisma.JsonValue | null;
@@ -759,6 +841,24 @@ export class FlyerImportService {
      return value.map((entry) => String(entry));
    };

+    const toSignals = (value: Prisma.JsonValue | null | undefined): ImportedItemSignals | null => {
+      if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+      const record = value as Record<string, unknown>;
+      const toArray = (key: string): string[] => {
+        const maybeArray = record[key];
+        if (!Array.isArray(maybeArray)) return [];
+        return maybeArray.map((entry) => String(entry));
+      };
+
+      return {
+        originCountries: toArray('originCountries'),
+        labels: toArray('labels'),
+        qualityFlags: toArray('qualityFlags'),
+        variant: typeof record.variant === 'string' ? record.variant : null,
+        packaging: typeof record.packaging === 'string' ? record.packaging : null,
+      };
+    };
+
    const normalizedMatchVia =
      item.matchedVia === 'alias' || item.matchedVia === 'exact' || item.matchedVia === 'token'
        ? item.matchedVia
@@ -784,6 +884,14 @@ export class FlyerImportService {
      bundleWeight: item.bundleWeight,
      isBundle: item.isBundle,
      bundleItems: this.sanitizeBundleItems(toStringArray(item.bundleItems)),
+      displayNameDetailed:
+        item.displayNameDetailed ??
+        buildDisplayNameDetailed({
+          rawName: item.rawName,
+          isBundle: item.isBundle,
+          bundleItems: this.sanitizeBundleItems(toStringArray(item.bundleItems)),
+        }),
+      signals: toSignals(item.signals),
      offerText: item.offerText,
      isOffer:
        item.price != null
@@ -858,6 +966,8 @@ export class FlyerImportService {
      bundleWeight: string | null;
      isBundle: boolean;
      bundleItems: Prisma.JsonValue | null;
+      displayNameDetailed?: string | null;
+      signals?: Prisma.JsonValue | null;
      offerText: string | null;
      parseConfidence: number;
      parseReasons: Prisma.JsonValue | null;
@@ -918,4 +1028,17 @@ export class FlyerImportService {
      .slice(0, this.MAX_BUNDLE_ITEMS)
      .map((entry) => entry.slice(0, this.MAX_BUNDLE_ITEM_LENGTH));
  }
+
+  private logImportMetrics(items: FlyerImportItem[]): void {
+    if (items.length === 0) return;
+
+    const noMatchCount = items.filter((item) => item.matchReasons.includes('no_match')).length;
+    const categoryAssignedCount = items.filter((item) => item.categoryId != null).length;
+    const noMatchRatio = (noMatchCount / items.length) * 100;
+    const categoryAssignedRatio = (categoryAssignedCount / items.length) * 100;
+
+    this.logger.log(
+      `Flyer import metrics: no_match=${noMatchCount}/${items.length} (${noMatchRatio.toFixed(1)}%), category_id=${categoryAssignedCount}/${items.length} (${categoryAssignedRatio.toFixed(1)}%)`,
+    );
+  }
 }