feat: F022 source acquisition, crawling & connector abstraction

This commit is contained in:
Benjamin Sutter
2026-05-15 15:32:53 +02:00
parent 8efd6e1974
commit fce0f684af
20 changed files with 1647 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
import type { DataSource, ConnectorRun, SourceStatus, TermsStatus, SourceFilters } from '../domain/dataSource'
export interface IDataSourceProvider {
getSources(filters?: SourceFilters): Promise<DataSource[]>
getSource(id: string): Promise<DataSource | null>
getConnectorRuns(sourceId: string): Promise<ConnectorRun[]>
triggerMockRun(sourceId: string): Promise<ConnectorRun>
updateSourceStatus(id: string, status: SourceStatus): Promise<DataSource>
markTermsStatus(id: string, termsStatus: TermsStatus): Promise<DataSource>
}
+75
View File
@@ -0,0 +1,75 @@
import type { IDataSourceProvider } from './IDataSourceProvider'
import type { DataSource, ConnectorRun, SourceStatus, TermsStatus, SourceFilters } from '../domain/dataSource'
import { ConnectorRunStatus } from '../domain/dataSource'
import { MOCK_DATA_SOURCES, MOCK_CONNECTOR_RUNS } from '../mock-data/dataSources'
let sources: DataSource[] = [...MOCK_DATA_SOURCES]
let runs: ConnectorRun[] = [...MOCK_CONNECTOR_RUNS]
function applyFilters(items: DataSource[], filters?: SourceFilters): DataSource[] {
if (!filters) return items
return items.filter((s) => {
if (filters.sourceType && s.sourceType !== filters.sourceType) return false
if (filters.status && s.status !== filters.status) return false
if (filters.termsStatus && s.termsStatus !== filters.termsStatus) return false
if (filters.search) {
const q = filters.search.toLowerCase()
if (!s.name.toLowerCase().includes(q) && !s.legalBasis.toLowerCase().includes(q)) return false
}
return true
})
}
export const MockupDataSourceProvider: IDataSourceProvider = {
async getSources(filters?: SourceFilters) {
return applyFilters([...sources], filters)
},
async getSource(id: string) {
return sources.find((s) => s.id === id) ?? null
},
async getConnectorRuns(sourceId: string) {
return runs
.filter((r) => r.sourceId === sourceId)
.sort((a, b) => b.startedAt.localeCompare(a.startedAt))
},
async triggerMockRun(sourceId: string) {
const source = sources.find((s) => s.id === sourceId)
const now = new Date().toISOString()
const newRun: ConnectorRun = {
id: `run-${crypto.randomUUID().slice(0, 8)}`,
sourceId,
startedAt: now,
finishedAt: now,
status: ConnectorRunStatus.COMPLETED,
itemsDetected: Math.floor(Math.random() * 200) + 50,
itemsNormalized: Math.floor(Math.random() * 180) + 40,
itemsRejected: Math.floor(Math.random() * 15),
signalsCreated: Math.floor(Math.random() * 8) + 1,
errors: [],
warnings: [],
runSummary: `Demo-Run für ${source?.name ?? sourceId} erfolgreich abgeschlossen.`,
}
runs = [newRun, ...runs]
sources = sources.map((s) =>
s.id === sourceId ? { ...s, lastRunAt: now } : s
)
return newRun
},
async updateSourceStatus(id: string, status: SourceStatus) {
const idx = sources.findIndex((s) => s.id === id)
if (idx === -1) throw new Error(`Source ${id} not found`)
sources[idx] = { ...sources[idx], status }
return sources[idx]
},
async markTermsStatus(id: string, termsStatus: TermsStatus) {
const idx = sources.findIndex((s) => s.id === id)
if (idx === -1) throw new Error(`Source ${id} not found`)
sources[idx] = { ...sources[idx], termsStatus }
return sources[idx]
},
}