feat: F022 source acquisition, crawling & connector abstraction

2026-05-15 15:32:53 +02:00
parent 8efd6e1974
commit fce0f684af
20 changed files with 1647 additions and 0 deletions
@@ -0,0 +1,302 @@
+import type { DataSource, ConnectorRun } from '../domain/dataSource'
+import {
+  DataSourceType,
+  SourceStatus,
+  TermsStatus,
+  ConnectorRunStatus,
+} from '../domain/dataSource'
+import { FreshnessStatus } from '../domain/enums'
+
+export const MOCK_DATA_SOURCES: DataSource[] = [
+  {
+    id: 'src-001',
+    name: 'ImmoScout24 – Zürich/Bern Feed',
+    sourceType: DataSourceType.PUBLIC_WEB_SOURCE,
+    legalBasis: 'Öffentlich zugängliche Listings – keine personenbezogenen Daten',
+    termsStatus: TermsStatus.NEEDS_LEGAL_REVIEW,
+    dataCategories: ['Gewerbeimmobilien', 'Mietpreise', 'Verfügbarkeit'],
+    supportedAssetTypes: ['OFFICE', 'RETAIL', 'LIGHT_INDUSTRIAL'],
+    regionCoverage: ['Zürich', 'Bern', 'Basel'],
+    reliabilityScore: 0.72,
+    freshnessStatus: FreshnessStatus.FRESH,
+    lastRunAt: '2026-05-15T06:00:00Z',
+    nextRunAt: '2026-05-16T06:00:00Z',
+    status: SourceStatus.ACTIVE,
+    notes: 'Daily refresh via FUTURE_CRAWLER_STUB. Rechtliche Freigabe ausstehend.',
+  },
+  {
+    id: 'src-002',
+    name: 'Ideal Sharing Portfolio Export',
+    sourceType: DataSourceType.INTERNAL_PORTFOLIO_EXPORT,
+    ownerOrganizationId: 'org-001',
+    legalBasis: 'Internes Dateneigentum – vollständig genehmigt',
+    termsStatus: TermsStatus.APPROVED,
+    dataCategories: ['Portoflio', 'Mietverträge', 'Flächen'],
+    supportedAssetTypes: ['OFFICE', 'LOGISTICS', 'MIXED'],
+    regionCoverage: ['Zürich', 'Zug', 'Luzern'],
+    reliabilityScore: 0.97,
+    freshnessStatus: FreshnessStatus.FRESH,
+    lastRunAt: '2026-05-15T02:00:00Z',
+    nextRunAt: '2026-05-15T14:00:00Z',
+    status: SourceStatus.ACTIVE,
+  },
+  {
+    id: 'src-003',
+    name: 'CBRE Market Data API',
+    sourceType: DataSourceType.API_CONNECTOR,
+    ownerOrganizationId: 'org-002',
+    legalBasis: 'API-Lizenzvertrag mit CBRE vom 12.01.2026',
+    termsStatus: TermsStatus.APPROVED,
+    dataCategories: ['Marktdaten', 'Mietindizes', 'Transaktionsvolumen'],
+    supportedAssetTypes: ['OFFICE', 'RETAIL', 'LOGISTICS'],
+    regionCoverage: ['Schweiz', 'DACH'],
+    reliabilityScore: 0.91,
+    freshnessStatus: FreshnessStatus.STALE,
+    lastRunAt: '2026-05-13T10:00:00Z',
+    nextRunAt: '2026-05-17T10:00:00Z',
+    status: SourceStatus.ACTIVE,
+    notes: 'Wöchentlicher Refresh – Quarterly Report verfügbar.',
+  },
+  {
+    id: 'src-004',
+    name: 'Handelsregister CH – Firmenumzüge',
+    sourceType: DataSourceType.PUBLIC_WEB_SOURCE,
+    legalBasis: 'Öffentliches Staatsregister (SHAB)',
+    termsStatus: TermsStatus.APPROVED,
+    dataCategories: ['Firmensitze', 'Umzüge', 'Neugründungen'],
+    supportedAssetTypes: ['OFFICE', 'MIXED'],
+    regionCoverage: ['Schweiz'],
+    reliabilityScore: 0.88,
+    freshnessStatus: FreshnessStatus.FRESH,
+    lastRunAt: '2026-05-15T03:30:00Z',
+    nextRunAt: '2026-05-16T03:30:00Z',
+    status: SourceStatus.ACTIVE,
+  },
+  {
+    id: 'src-005',
+    name: 'Baubewilligungsregister Kt. Zürich',
+    sourceType: DataSourceType.FUTURE_CRAWLER_STUB,
+    legalBasis: 'Amtliche Publikation – öffentlich zugänglich',
+    termsStatus: TermsStatus.UNKNOWN,
+    dataCategories: ['Baubewilligungen', 'Umnutzungen', 'Abbrüche'],
+    supportedAssetTypes: ['OFFICE', 'PRODUCTION', 'LIGHT_INDUSTRIAL'],
+    regionCoverage: ['Kanton Zürich'],
+    reliabilityScore: 0.65,
+    freshnessStatus: FreshnessStatus.OUTDATED,
+    lastRunAt: '2026-04-30T08:00:00Z',
+    status: SourceStatus.PENDING_REVIEW,
+    notes: 'Crawler-Implementierung noch ausstehend. Manuelle Prüfung erforderlich.',
+  },
+  {
+    id: 'src-006',
+    name: 'Mietvertragsdaten (CSV Upload – Q1 2026)',
+    sourceType: DataSourceType.CSV_IMPORT,
+    ownerOrganizationId: 'org-001',
+    legalBasis: 'Interner Upload durch Portfoliomanager – DSGVO-konform',
+    termsStatus: TermsStatus.APPROVED,
+    dataCategories: ['Mietverträge', 'Laufzeiten', 'Mieter'],
+    supportedAssetTypes: ['OFFICE', 'RETAIL'],
+    regionCoverage: ['Zürich', 'Basel'],
+    reliabilityScore: 0.84,
+    freshnessStatus: FreshnessStatus.STALE,
+    lastRunAt: '2026-03-31T09:00:00Z',
+    status: SourceStatus.PAUSED,
+    notes: 'Q2-Upload ausstehend. Quelle pausiert bis neue Datei verfügbar.',
+  },
+  {
+    id: 'src-007',
+    name: 'JLL Research Partner Feed',
+    sourceType: DataSourceType.PARTNER_FEED,
+    ownerOrganizationId: 'org-003',
+    legalBasis: 'Datenaustauschabkommen mit JLL Schweiz AG – vertraulich',
+    termsStatus: TermsStatus.RESTRICTED,
+    dataCategories: ['Marktberichte', 'Leerstandsquoten', 'Prime Rents'],
+    supportedAssetTypes: ['OFFICE', 'LOGISTICS'],
+    regionCoverage: ['Zürich', 'Genf', 'Basel'],
+    reliabilityScore: 0.93,
+    freshnessStatus: FreshnessStatus.FRESH,
+    lastRunAt: '2026-05-15T07:00:00Z',
+    nextRunAt: '2026-05-22T07:00:00Z',
+    status: SourceStatus.ACTIVE,
+    notes: 'Wöchentlicher Push durch JLL. Nur für interne Nutzung – kein Re-Export.',
+  },
+  {
+    id: 'src-008',
+    name: 'Analyst-Eingaben (Team Zürich)',
+    sourceType: DataSourceType.ANALYST_ENTRY,
+    ownerOrganizationId: 'org-001',
+    legalBasis: 'Interne Datenerhebung durch Analysten-Team',
+    termsStatus: TermsStatus.APPROVED,
+    dataCategories: ['Marktsignale', 'Qualitative Einschätzungen', 'Netzwerkinfos'],
+    supportedAssetTypes: ['OFFICE', 'RETAIL', 'MIXED'],
+    regionCoverage: ['Zürich', 'Winterthur'],
+    reliabilityScore: 0.78,
+    freshnessStatus: FreshnessStatus.FRESH,
+    lastRunAt: '2026-05-14T16:45:00Z',
+    status: SourceStatus.ACTIVE,
+  },
+  {
+    id: 'src-009',
+    name: 'Homegate Gewerbe-Scraper (Beta)',
+    sourceType: DataSourceType.FUTURE_CRAWLER_STUB,
+    legalBasis: 'In rechtlicher Prüfung – noch nicht freigegeben',
+    termsStatus: TermsStatus.BLOCKED,
+    dataCategories: ['Gewerbelistings', 'Mietpreise'],
+    supportedAssetTypes: ['OFFICE', 'RETAIL'],
+    regionCoverage: ['Schweiz'],
+    reliabilityScore: 0.0,
+    freshnessStatus: FreshnessStatus.OUTDATED,
+    status: SourceStatus.DISABLED,
+    errorState: 'Quelle gesperrt – robots.txt-Prüfung negativ, AGB untersagt automatisiertes Crawling.',
+  },
+]
+
+export const MOCK_CONNECTOR_RUNS: ConnectorRun[] = [
+  // src-001 runs
+  {
+    id: 'run-001-a',
+    sourceId: 'src-001',
+    startedAt: '2026-05-15T06:00:00Z',
+    finishedAt: '2026-05-15T06:12:34Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 342,
+    itemsNormalized: 318,
+    itemsRejected: 24,
+    signalsCreated: 7,
+    errors: [],
+    warnings: ['24 Einträge ohne gültige Flächenangabe übersprungen'],
+    runSummary: '342 Listings importiert. 318 normalisiert, 7 neue Marktsignale erkannt.',
+  },
+  {
+    id: 'run-001-b',
+    sourceId: 'src-001',
+    startedAt: '2026-05-14T06:00:00Z',
+    finishedAt: '2026-05-14T06:09:11Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 289,
+    itemsNormalized: 277,
+    itemsRejected: 12,
+    signalsCreated: 4,
+    errors: [],
+    warnings: [],
+    runSummary: '289 Listings importiert. 4 neue Marktsignale erkannt.',
+  },
+  {
+    id: 'run-001-c',
+    sourceId: 'src-001',
+    startedAt: '2026-05-13T06:00:00Z',
+    finishedAt: '2026-05-13T06:04:22Z',
+    status: ConnectorRunStatus.PARTIAL,
+    itemsDetected: 310,
+    itemsNormalized: 201,
+    itemsRejected: 109,
+    signalsCreated: 2,
+    errors: ['HTTP 429 nach 201 Einträgen – Rate Limit erreicht'],
+    warnings: ['109 Einträge konnten nicht abgerufen werden'],
+    runSummary: 'Teilimport wegen Rate Limiting. 201 von 310 Einträgen verarbeitet.',
+  },
+  // src-002 runs
+  {
+    id: 'run-002-a',
+    sourceId: 'src-002',
+    startedAt: '2026-05-15T02:00:00Z',
+    finishedAt: '2026-05-15T02:03:08Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 1240,
+    itemsNormalized: 1240,
+    itemsRejected: 0,
+    signalsCreated: 12,
+    errors: [],
+    warnings: [],
+    runSummary: 'Vollständiger Portfolio-Sync. 12 Lease-Expiry-Signale erzeugt.',
+  },
+  {
+    id: 'run-002-b',
+    sourceId: 'src-002',
+    startedAt: '2026-05-14T14:00:00Z',
+    finishedAt: '2026-05-14T14:02:55Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 1238,
+    itemsNormalized: 1238,
+    itemsRejected: 0,
+    signalsCreated: 3,
+    errors: [],
+    warnings: [],
+    runSummary: 'Delta-Sync erfolgreich. 3 neue Objekte hinzugefügt.',
+  },
+  // src-003 runs
+  {
+    id: 'run-003-a',
+    sourceId: 'src-003',
+    startedAt: '2026-05-13T10:00:00Z',
+    finishedAt: '2026-05-13T10:18:42Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 86,
+    itemsNormalized: 86,
+    itemsRejected: 0,
+    signalsCreated: 0,
+    errors: [],
+    warnings: ['Quartalsdaten verfügbar – manueller Review empfohlen'],
+    runSummary: 'Marktdaten-Update Q1 2026 importiert. 86 Datenpunkte aktualisiert.',
+  },
+  // src-004 runs
+  {
+    id: 'run-004-a',
+    sourceId: 'src-004',
+    startedAt: '2026-05-15T03:30:00Z',
+    finishedAt: '2026-05-15T03:44:17Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 523,
+    itemsNormalized: 498,
+    itemsRejected: 25,
+    signalsCreated: 9,
+    errors: [],
+    warnings: ['25 Einträge ohne Adressangabe ignoriert'],
+    runSummary: '523 Handelsregistereinträge geprüft. 9 Firmensitz-Signale erkannt.',
+  },
+  // src-007 runs
+  {
+    id: 'run-007-a',
+    sourceId: 'src-007',
+    startedAt: '2026-05-15T07:00:00Z',
+    finishedAt: '2026-05-15T07:05:30Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 14,
+    itemsNormalized: 14,
+    itemsRejected: 0,
+    signalsCreated: 3,
+    errors: [],
+    warnings: [],
+    runSummary: 'JLL Weekly Report verarbeitet. 3 Marktberichte als Signale importiert.',
+  },
+  // src-008 runs
+  {
+    id: 'run-008-a',
+    sourceId: 'src-008',
+    startedAt: '2026-05-14T16:45:00Z',
+    finishedAt: '2026-05-14T16:45:52Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 3,
+    itemsNormalized: 3,
+    itemsRejected: 0,
+    signalsCreated: 3,
+    errors: [],
+    warnings: [],
+    runSummary: '3 manuelle Analysten-Einträge verarbeitet. Je 1 Signal pro Eintrag.',
+  },
+  // src-006 – last run before pause
+  {
+    id: 'run-006-a',
+    sourceId: 'src-006',
+    startedAt: '2026-03-31T09:00:00Z',
+    finishedAt: '2026-03-31T09:02:19Z',
+    status: ConnectorRunStatus.COMPLETED,
+    itemsDetected: 412,
+    itemsNormalized: 408,
+    itemsRejected: 4,
+    signalsCreated: 18,
+    errors: [],
+    warnings: ['4 Einträge mit doppelter Vertrags-ID ignoriert'],
+    runSummary: 'Q1 CSV verarbeitet. 408 Mietverträge importiert, 18 Lease-Expiry-Signale.',
+  },
+]