feat: detect duplicates in csv
This commit is contained in:
+148
-90
@@ -121,6 +121,129 @@ function handle_post(): never
|
||||
$mode = 'clear';
|
||||
}
|
||||
|
||||
// Resolutions map: mobile_number => chosen group index (0-based)
|
||||
// Sent on the second submission after the user resolves duplicates in the UI.
|
||||
$resolutions = [];
|
||||
if (!empty($_POST['resolutions'])) {
|
||||
$dec = json_decode((string) $_POST['resolutions'], true);
|
||||
if (is_array($dec)) {
|
||||
$resolutions = $dec;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Phase 1: Parse all valid rows into memory ────────────────────────────
|
||||
|
||||
$validRows = [];
|
||||
$skipped = 0;
|
||||
$skipReasons = ['not_attending' => 0, 'no_mobile' => 0];
|
||||
$rejected = [];
|
||||
|
||||
while (($row = fgetcsv($handle, 0, $delim)) !== false) {
|
||||
$row = array_map(fn($v) => trim((string) $v, " \t\n\r\0\x0B\""), $row);
|
||||
|
||||
$firstName = trim((string) ($row[$col['first_name']] ?? ''));
|
||||
$lastName = trim((string) ($row[$col['last_name']] ?? ''));
|
||||
$email = trim((string) ($row[$col['email']] ?? ''));
|
||||
$rawMobile = trim((string) ($row[$col['mobile_number']] ?? ''), " \t\n\r\0\x0B\"'");
|
||||
$rawStatus = $col['status'] !== null ? trim((string) ($row[$col['status']] ?? '')) : '';
|
||||
|
||||
if ($col['status'] !== null) {
|
||||
$status = strtolower($rawStatus);
|
||||
if ($status !== '' && $status !== 'attending') {
|
||||
$skipped++;
|
||||
$skipReasons['not_attending']++;
|
||||
$rejected[] = [
|
||||
'first_name' => $firstName, 'last_name' => $lastName,
|
||||
'email' => $email, 'raw_mobile' => $rawMobile,
|
||||
'status' => $rawStatus, 'reason' => 'not_attending',
|
||||
];
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
$mobile = sanitise_mobile($rawMobile);
|
||||
if ($mobile === '') {
|
||||
$skipped++;
|
||||
$skipReasons['no_mobile']++;
|
||||
$rejected[] = [
|
||||
'first_name' => $firstName, 'last_name' => $lastName,
|
||||
'email' => $email, 'raw_mobile' => $rawMobile,
|
||||
'status' => $rawStatus, 'reason' => classify_mobile_rejection($rawMobile),
|
||||
];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse workshop columns up-front (needed for duplicate display and import)
|
||||
$wsName = $hasWorkshops ? trim((string) ($row[$col['workshop_name']] ?? '')) : '';
|
||||
$wsTime = ($hasWorkshops && $col['workshop_time'] !== null)
|
||||
? trim((string) ($row[$col['workshop_time']] ?? ''))
|
||||
: '';
|
||||
$dt = null;
|
||||
if ($hasWorkshops && $wsName !== '') {
|
||||
if ($wsTime === '') {
|
||||
$wsTime = extract_time_from_string($wsName) ?? '';
|
||||
$wsName = trim_time_from_string($wsName);
|
||||
}
|
||||
if ($wsTime !== '') {
|
||||
$parsed = strtotime($wsTime);
|
||||
$dt = $parsed !== false ? date('Y-m-d H:i:s', $parsed) : null;
|
||||
}
|
||||
}
|
||||
|
||||
$validRows[] = [
|
||||
'first_name' => $firstName,
|
||||
'last_name' => $lastName,
|
||||
'email' => $email,
|
||||
'mobile_number' => $mobile,
|
||||
'workshop_name' => $wsName,
|
||||
'workshop_time' => $dt,
|
||||
];
|
||||
}
|
||||
fclose($handle);
|
||||
|
||||
// ─── Phase 2: Detect intra-CSV duplicate mobile numbers ───────────────────
|
||||
|
||||
$byMobile = [];
|
||||
foreach ($validRows as $i => $vr) {
|
||||
$byMobile[$vr['mobile_number']][] = $i;
|
||||
}
|
||||
|
||||
$duplicateGroups = [];
|
||||
foreach ($byMobile as $mobile => $indices) {
|
||||
if (count($indices) > 1) {
|
||||
$duplicateGroups[] = [
|
||||
'mobile' => $mobile,
|
||||
'rows' => array_values(array_map(fn($i) => $validRows[$i], $indices)),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// If duplicates exist and the user hasn't resolved them yet, pause and ask.
|
||||
if (!empty($duplicateGroups) && empty($resolutions)) {
|
||||
json_out(['status' => 'needs_resolution', 'duplicates' => $duplicateGroups]);
|
||||
}
|
||||
|
||||
// Apply resolutions — keep only the chosen row for each duplicate mobile.
|
||||
if (!empty($resolutions)) {
|
||||
$indicesToRemove = [];
|
||||
foreach ($byMobile as $mobile => $indices) {
|
||||
if (!isset($resolutions[$mobile]) || count($indices) < 2) {
|
||||
continue;
|
||||
}
|
||||
$chosenGroupPos = (int) $resolutions[$mobile];
|
||||
foreach ($indices as $groupPos => $rowIdx) {
|
||||
if ($groupPos !== $chosenGroupPos) {
|
||||
$indicesToRemove[$rowIdx] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
$validRows = array_values(
|
||||
array_filter($validRows, fn($_, $i) => !isset($indicesToRemove[$i]), ARRAY_FILTER_USE_BOTH)
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Phase 3: Import ──────────────────────────────────────────────────────
|
||||
|
||||
$db = db();
|
||||
$db->beginTransaction();
|
||||
|
||||
@@ -132,7 +255,7 @@ function handle_post(): never
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-populate workshop cache from existing sessions (for overwrite/append)
|
||||
// Pre-populate workshop cache from existing sessions (for overwrite mode)
|
||||
$workshopCache = [];
|
||||
if ($hasWorkshops && $mode !== 'clear') {
|
||||
foreach ($db->query('SELECT id, workshop_name, workshop_time FROM workshop_sessions')->fetchAll() as $ws) {
|
||||
@@ -140,76 +263,39 @@ function handle_post(): never
|
||||
}
|
||||
}
|
||||
|
||||
$workshopStmt = $db->prepare(
|
||||
$workshopStmt = $db->prepare(
|
||||
'INSERT INTO workshop_sessions (workshop_name, workshop_time) VALUES (:name, :time)'
|
||||
);
|
||||
$regStmt = $db->prepare(
|
||||
$regStmt = $db->prepare(
|
||||
'INSERT INTO registrations (attendee_id, workshop_session_id) VALUES (:attendee_id, :ws_id)'
|
||||
);
|
||||
$insertStmt = $db->prepare(
|
||||
$insertStmt = $db->prepare(
|
||||
'INSERT INTO attendees (first_name, last_name, email, mobile_number)
|
||||
VALUES (:first_name, :last_name, :email, :mobile_number)'
|
||||
VALUES (:first_name, :last_name, :email, :mobile_number)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
first_name = VALUES(first_name),
|
||||
last_name = VALUES(last_name),
|
||||
email = VALUES(email),
|
||||
id = LAST_INSERT_ID(id)'
|
||||
);
|
||||
// Match by email — most stable identifier
|
||||
$findByEmailStmt = $db->prepare(
|
||||
"SELECT id FROM attendees WHERE email = :email AND email != ''"
|
||||
);
|
||||
// Update all fields including mobile (e.g. number may have changed, matched by email)
|
||||
$updateStmt = $db->prepare(
|
||||
$updateStmt = $db->prepare(
|
||||
'UPDATE attendees SET first_name=:first_name, last_name=:last_name, email=:email, mobile_number=:mobile_number WHERE id=:id'
|
||||
);
|
||||
$deleteRegsStmt = $db->prepare(
|
||||
$deleteRegsStmt = $db->prepare(
|
||||
'DELETE FROM registrations WHERE attendee_id = :attendee_id'
|
||||
);
|
||||
|
||||
$imported = 0;
|
||||
$updated = 0;
|
||||
$skipped = 0;
|
||||
$skipReasons = ['not_attending' => 0, 'no_mobile' => 0];
|
||||
$rejected = [];
|
||||
|
||||
while (($row = fgetcsv($handle, 0, $delim)) !== false) {
|
||||
// Strip leading/trailing double-quotes that some spreadsheet exports add around field values
|
||||
$row = array_map(fn($v) => trim((string) $v, " \t\n\r\0\x0B\""), $row);
|
||||
|
||||
$firstName = trim((string) ($row[$col['first_name']] ?? ''));
|
||||
$lastName = trim((string) ($row[$col['last_name']] ?? ''));
|
||||
$email = trim((string) ($row[$col['email']] ?? ''));
|
||||
$rawMobile = trim((string) ($row[$col['mobile_number']] ?? ''), " \t\n\r\0\x0B\"'");
|
||||
$rawStatus = $col['status'] !== null ? trim((string) ($row[$col['status']] ?? '')) : '';
|
||||
|
||||
// Skip non-attending rows when a status column is present
|
||||
if ($col['status'] !== null) {
|
||||
$status = strtolower($rawStatus);
|
||||
if ($status !== '' && $status !== 'attending') {
|
||||
$skipped++;
|
||||
$skipReasons['not_attending']++;
|
||||
$rejected[] = [
|
||||
'first_name' => $firstName,
|
||||
'last_name' => $lastName,
|
||||
'email' => $email,
|
||||
'raw_mobile' => $rawMobile,
|
||||
'status' => $rawStatus,
|
||||
'reason' => 'not_attending',
|
||||
];
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
$mobile = sanitise_mobile($rawMobile);
|
||||
if ($mobile === '') {
|
||||
$skipped++;
|
||||
$skipReasons['no_mobile']++;
|
||||
$rejected[] = [
|
||||
'first_name' => $firstName,
|
||||
'last_name' => $lastName,
|
||||
'email' => $email,
|
||||
'raw_mobile' => $rawMobile,
|
||||
'status' => $rawStatus,
|
||||
'reason' => classify_mobile_rejection($rawMobile),
|
||||
];
|
||||
continue;
|
||||
}
|
||||
foreach ($validRows as $vr) {
|
||||
$firstName = $vr['first_name'];
|
||||
$lastName = $vr['last_name'];
|
||||
$email = $vr['email'];
|
||||
$mobile = $vr['mobile_number'];
|
||||
|
||||
if ($mode === 'clear') {
|
||||
$insertStmt->execute([
|
||||
@@ -219,9 +305,9 @@ function handle_post(): never
|
||||
':mobile_number' => $mobile,
|
||||
]);
|
||||
$attendeeId = (int) $db->lastInsertId();
|
||||
$imported++;
|
||||
// rowCount: 1 = new insert, 2 = ON DUPLICATE KEY UPDATE fired
|
||||
if ($insertStmt->rowCount() >= 2) { $updated++; } else { $imported++; }
|
||||
} else {
|
||||
// Match by email only
|
||||
$matches = [];
|
||||
if ($email !== '') {
|
||||
$findByEmailStmt->execute([':email' => $email]);
|
||||
@@ -229,7 +315,6 @@ function handle_post(): never
|
||||
}
|
||||
|
||||
if (count($matches) === 0) {
|
||||
// Not found — insert
|
||||
$insertStmt->execute([
|
||||
':first_name' => $firstName,
|
||||
':last_name' => $lastName,
|
||||
@@ -239,7 +324,6 @@ function handle_post(): never
|
||||
$attendeeId = (int) $db->lastInsertId();
|
||||
$imported++;
|
||||
} else {
|
||||
// overwrite: update all fields (incl. mobile), collapse any duplicates
|
||||
$attendeeId = (int) $matches[0]['id'];
|
||||
$updateStmt->execute([
|
||||
':first_name' => $firstName,
|
||||
@@ -259,42 +343,16 @@ function handle_post(): never
|
||||
}
|
||||
}
|
||||
|
||||
// Assign to a workshop session (runs for all modes)
|
||||
if ($hasWorkshops) {
|
||||
$wsName = trim((string) ($row[$col['workshop_name']] ?? ''));
|
||||
|
||||
if ($wsName !== '') {
|
||||
// Try separate time column first; otherwise extract time from the name
|
||||
$wsTime = $col['workshop_time'] !== null
|
||||
? trim((string) ($row[$col['workshop_time']] ?? ''))
|
||||
: '';
|
||||
|
||||
if ($wsTime === '') {
|
||||
$wsTime = extract_time_from_string($wsName) ?? '';
|
||||
// Strip the extracted time token from the workshop name
|
||||
$wsName = trim_time_from_string($wsName);
|
||||
}
|
||||
|
||||
$dt = null;
|
||||
if ($wsTime !== '') {
|
||||
$parsed = strtotime($wsTime);
|
||||
$dt = $parsed !== false ? date('Y-m-d H:i:s', $parsed) : null;
|
||||
}
|
||||
|
||||
$cacheKey = $wsName . '|' . ($dt ?? '');
|
||||
if (!isset($workshopCache[$cacheKey])) {
|
||||
$workshopStmt->execute([':name' => $wsName, ':time' => $dt]);
|
||||
$workshopCache[$cacheKey] = (int) $db->lastInsertId();
|
||||
}
|
||||
$regStmt->execute([
|
||||
':attendee_id' => $attendeeId,
|
||||
':ws_id' => $workshopCache[$cacheKey],
|
||||
]);
|
||||
if ($hasWorkshops && $vr['workshop_name'] !== '') {
|
||||
$cacheKey = $vr['workshop_name'] . '|' . ($vr['workshop_time'] ?? '');
|
||||
if (!isset($workshopCache[$cacheKey])) {
|
||||
$workshopStmt->execute([':name' => $vr['workshop_name'], ':time' => $vr['workshop_time']]);
|
||||
$workshopCache[$cacheKey] = (int) $db->lastInsertId();
|
||||
}
|
||||
$regStmt->execute([':attendee_id' => $attendeeId, ':ws_id' => $workshopCache[$cacheKey]]);
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
$db->commit();
|
||||
|
||||
json_out(['imported' => $imported, 'updated' => $updated, 'skipped' => $skipped, 'skip_reasons' => $skipReasons, 'workshops_imported' => $hasWorkshops, 'rejected' => $rejected]);
|
||||
|
||||
Reference in New Issue
Block a user