-
-
Notifications
You must be signed in to change notification settings - Fork 0
Add multi-inbox support to backend #10
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,7 @@ import ( | |
| "log" | ||
| "os" | ||
| "os/signal" | ||
| "sync" | ||
| "syscall" | ||
| "time" | ||
|
|
||
|
|
@@ -167,50 +168,104 @@ func run(ctx context.Context, cmd *cli.Command) error { | |
| func fetchReports(cfg *config.Config, store *storage.Storage) error { | ||
| log.Println("Fetching DMARC reports...") | ||
|
|
||
| // Create IMAP client | ||
| client := imap.NewClient(&cfg.IMAP) | ||
| if err := client.Connect(); err != nil { | ||
| return err | ||
| // Get all IMAP configurations (supports both single and multiple inboxes) | ||
| imapConfigs := cfg.GetIMAPConfigs() | ||
| if len(imapConfigs) == 0 { | ||
| return fmt.Errorf("no IMAP configuration found") | ||
| } | ||
| defer func() { _ = client.Disconnect() }() | ||
|
|
||
| // Fetch reports | ||
| reports, err := client.FetchDMARCReports() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| log.Printf("Fetching from %d inbox(es)...", len(imapConfigs)) | ||
|
|
||
| if len(reports) == 0 { | ||
| log.Println("No new reports found") | ||
| return nil | ||
| } | ||
| var wg sync.WaitGroup | ||
| var mu sync.Mutex | ||
| var errors []error | ||
| totalProcessed := 0 | ||
|
|
||
| // Fetch from each IMAP inbox concurrently | ||
| for i := range imapConfigs { | ||
| wg.Add(1) | ||
| go func(imapCfg *config.IMAPConfig, index int) { | ||
|
||
| defer wg.Done() | ||
|
|
||
| log.Printf("Processing %d reports...", len(reports)) | ||
| mailboxName := fmt.Sprintf("%s@%s:%s", imapCfg.Username, imapCfg.Host, imapCfg.Mailbox) | ||
| log.Printf("[Inbox %d/%d] Connecting to %s", index+1, len(imapConfigs), mailboxName) | ||
|
|
||
| // Process each report | ||
| processed := 0 | ||
| for _, report := range reports { | ||
| for _, attachment := range report.Attachments { | ||
| feedback, err := parser.ParseReport(attachment.Data) | ||
| // Create IMAP client for this inbox | ||
| client := imap.NewClient(imapCfg) | ||
| if err := client.Connect(); err != nil { | ||
| mu.Lock() | ||
| errors = append(errors, fmt.Errorf("failed to connect to %s: %w", mailboxName, err)) | ||
| mu.Unlock() | ||
| return | ||
| } | ||
| defer func() { _ = client.Disconnect() }() | ||
|
|
||
| // Fetch reports from this inbox | ||
| reports, err := client.FetchDMARCReports() | ||
| if err != nil { | ||
| log.Printf("Failed to parse %s: %v", attachment.Filename, err) | ||
| continue | ||
| mu.Lock() | ||
| errors = append(errors, fmt.Errorf("failed to fetch from %s: %w", mailboxName, err)) | ||
| mu.Unlock() | ||
| return | ||
| } | ||
|
|
||
| if err := store.SaveReport(feedback); err != nil { | ||
| log.Printf("Failed to save report %s: %v", feedback.ReportMetadata.ReportID, err) | ||
| continue | ||
| if len(reports) == 0 { | ||
| log.Printf("[Inbox %d/%d] No new reports found in %s", index+1, len(imapConfigs), mailboxName) | ||
| return | ||
| } | ||
|
|
||
| log.Printf("Saved report: %s from %s (domain: %s, messages: %d)", | ||
| feedback.ReportMetadata.ReportID, | ||
| feedback.ReportMetadata.OrgName, | ||
| feedback.PolicyPublished.Domain, | ||
| feedback.GetTotalMessages()) | ||
| processed++ | ||
| log.Printf("[Inbox %d/%d] Processing %d report(s) from %s", index+1, len(imapConfigs), len(reports), mailboxName) | ||
|
|
||
| // Process each report | ||
| processed := 0 | ||
| for _, report := range reports { | ||
| for _, attachment := range report.Attachments { | ||
| feedback, err := parser.ParseReport(attachment.Data) | ||
| if err != nil { | ||
| log.Printf("[Inbox %d/%d] Failed to parse %s: %v", index+1, len(imapConfigs), attachment.Filename, err) | ||
| continue | ||
| } | ||
|
|
||
| if err := store.SaveReport(feedback); err != nil { | ||
| log.Printf("[Inbox %d/%d] Failed to save report %s: %v", index+1, len(imapConfigs), feedback.ReportMetadata.ReportID, err) | ||
| continue | ||
| } | ||
|
|
||
| log.Printf("[Inbox %d/%d] Saved report: %s from %s (domain: %s, messages: %d)", | ||
| index+1, len(imapConfigs), | ||
| feedback.ReportMetadata.ReportID, | ||
| feedback.ReportMetadata.OrgName, | ||
| feedback.PolicyPublished.Domain, | ||
| feedback.GetTotalMessages()) | ||
| processed++ | ||
| } | ||
| } | ||
|
|
||
| mu.Lock() | ||
| totalProcessed += processed | ||
| mu.Unlock() | ||
|
|
||
| log.Printf("[Inbox %d/%d] Successfully processed %d report(s) from %s", index+1, len(imapConfigs), processed, mailboxName) | ||
| }(&imapConfigs[i], i) | ||
| } | ||
|
|
||
| // Wait for all fetches to complete | ||
| wg.Wait() | ||
|
|
||
| // Report results | ||
| if len(errors) > 0 { | ||
| log.Printf("Completed with %d error(s):", len(errors)) | ||
| for _, err := range errors { | ||
| log.Printf(" - %v", err) | ||
| } | ||
| } | ||
|
|
||
| log.Printf("Successfully processed %d reports", processed) | ||
| log.Printf("Successfully processed %d total report(s) across all inboxes", totalProcessed) | ||
|
|
||
| // Return error only if all fetches failed | ||
| if len(errors) > 0 && len(errors) == len(imapConfigs) { | ||
| return fmt.Errorf("all inbox fetches failed") | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| { | ||
| "database": { | ||
| "path": "~/.parse-dmarc/db.sqlite" | ||
| }, | ||
| "imap_configs": [ | ||
| { | ||
| "host": "imap.gmail.com", | ||
| "mailbox": "INBOX", | ||
| "password": "your-app-password-1", | ||
| "port": 993, | ||
| "use_tls": true, | ||
| "username": "account1@gmail.com" | ||
| }, | ||
| { | ||
| "host": "imap.gmail.com", | ||
| "mailbox": "INBOX", | ||
| "password": "your-app-password-2", | ||
| "port": 993, | ||
| "use_tls": true, | ||
| "username": "account2@gmail.com" | ||
| }, | ||
| { | ||
| "host": "imap.example.com", | ||
| "mailbox": "DMARC", | ||
| "password": "your-password-3", | ||
| "port": 993, | ||
| "use_tls": true, | ||
| "username": "account3@example.com" | ||
| } | ||
| ], | ||
| "server": { | ||
| "host": "0.0.0.0", | ||
| "port": 8080 | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,9 +10,10 @@ import ( | |
|
|
||
| // Config holds the application configuration | ||
| type Config struct { | ||
| IMAP IMAPConfig `json:"imap"` | ||
| Database DatabaseConfig `json:"database"` | ||
| Server ServerConfig `json:"server"` | ||
| IMAP IMAPConfig `json:"imap"` | ||
| IMAPConfigs []IMAPConfig `json:"imap_configs"` | ||
| Database DatabaseConfig `json:"database"` | ||
| Server ServerConfig `json:"server"` | ||
| } | ||
|
|
||
| // IMAPConfig holds IMAP server configuration | ||
|
|
@@ -36,6 +37,21 @@ type ServerConfig struct { | |
| Host string `json:"host" env:"SERVER_HOST" envDefault:"0.0.0.0"` | ||
| } | ||
|
|
||
| // GetIMAPConfigs returns all IMAP configurations, normalizing single and multiple configs | ||
| func (c *Config) GetIMAPConfigs() []IMAPConfig { | ||
| // If IMAPConfigs array is populated, use it | ||
| if len(c.IMAPConfigs) > 0 { | ||
| return c.IMAPConfigs | ||
| } | ||
|
|
||
| // Otherwise, use the single IMAP config for backward compatibility | ||
| if c.IMAP.Host != "" { | ||
| return []IMAPConfig{c.IMAP} | ||
| } | ||
|
|
||
| return []IMAPConfig{} | ||
| } | ||
|
|
||
| func defaultDBPath() (string, error) { | ||
| home, err := os.UserHomeDir() | ||
| if err != nil { | ||
|
|
@@ -69,15 +85,48 @@ func Load(path string) (*Config, error) { | |
| } | ||
| } | ||
|
|
||
| if err := env.Parse(&cfg); err != nil { | ||
| return nil, err | ||
| // Support IMAP_CONFIGS environment variable for multiple configs | ||
| imapConfigsJSON := os.Getenv("IMAP_CONFIGS") | ||
| if imapConfigsJSON != "" { | ||
| var imapConfigs []IMAPConfig | ||
| if err := json.Unmarshal([]byte(imapConfigsJSON), &imapConfigs); err != nil { | ||
| return nil, err | ||
| } | ||
| cfg.IMAPConfigs = imapConfigs | ||
| } | ||
|
Comment on lines
+88
to
+96
|
||
|
|
||
| // Only parse environment variables if we're using single IMAP config | ||
| // (env.Parse incorrectly applies envDefault to array elements) | ||
| if len(cfg.IMAPConfigs) == 0 { | ||
| if err := env.Parse(&cfg); err != nil { | ||
| return nil, err | ||
| } | ||
| } else { | ||
| // Parse only non-IMAP fields to avoid envDefault overwriting array values | ||
| if err := env.Parse(&cfg.Database); err != nil { | ||
| return nil, err | ||
| } | ||
| if err := env.Parse(&cfg.Server); err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| if cfg.IMAP.Port == 0 { | ||
| cfg.IMAP.Port = 993 | ||
| // Apply defaults to all IMAP configs | ||
| allConfigs := cfg.GetIMAPConfigs() | ||
| for i := range allConfigs { | ||
| if allConfigs[i].Port == 0 { | ||
| allConfigs[i].Port = 993 | ||
| } | ||
| if allConfigs[i].Mailbox == "" { | ||
| allConfigs[i].Mailbox = "INBOX" | ||
| } | ||
| } | ||
|
Comment on lines
+115
to
123
|
||
| if cfg.IMAP.Mailbox == "" { | ||
| cfg.IMAP.Mailbox = "INBOX" | ||
|
|
||
| // Update the config with normalized values | ||
| if len(cfg.IMAPConfigs) > 0 { | ||
| cfg.IMAPConfigs = allConfigs | ||
| } else if cfg.IMAP.Host != "" { | ||
| cfg.IMAP = allConfigs[0] | ||
|
Comment on lines
+128
to
+129
|
||
| } | ||
| if cfg.Database.Path == "" { | ||
| cfg.Database.Path, err = defaultDBPath() | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The documentation claims 'Automatic deduplication of reports across inboxes', but this feature relies on the existing
INSERT OR IGNOREbehavior inSaveReport()which usesreport_idas a unique constraint. This is not new functionality added by this PR. Consider clarifying that deduplication is handled by the existing database unique constraint onreport_id, or remove this bullet if it's not specific to the multi-inbox feature.