Incidents

Timeline Events Developer Guide

This guide shows developers how to integrate timeline events into platform components, connectors, and custom services.

Overview

The Timeline Service uses CloudEvents v1.0 as the standard event format, with incident management extensions for metadata and policy enforcement.

Architecture 

┌──────────────┐      CloudEvents      ┌──────────────────┐
│   Your       │────────────────────►  │  Timeline        │
│   Service    │   (append event)      │  Service         │
└──────────────┘                       └────────┬─────────┘
                                                │
                                                │ (query/export)
                                                ▼
                                       ┌──────────────────┐
                                       │  Event Bus       │
                                       │  (notifications) │
                                       └──────────────────┘

Quick Start

Install the Go SDK 

go get github.com/systmms/incidents

Import Required Packages 

import (
    "github.com/systmms/incidents/internal/timeline"
    "github.com/systmms/incidents/internal/models"
    "github.com/systmms/incidents/internal/storage"
)

Initialize the Timeline Service 

func main() {
    // Initialize database connection
    db, err := storage.New("postgres://user:pass@localhost/incidents")
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()

    // Create timeline service
    svc := timeline.New(db)

    // Use the service to append/query events
    // ... (see examples below)
}

Emitting Events

Basic Event Append 

func declareIncident(svc *timeline.Service, incidentID string) error {
    // Create timeline event
    event := &models.TimelineEvent{
        IncidentID: incidentID,
        EventType:  models.EventTypeDeclare,
        Actor:      "alice@example.com",
        Title:      "Database Connection Pool Exhausted",
        Content:    "Payment API experiencing connection timeouts",
        EventTime:  time.Now(),
        Source:     "api",
        Data: map[string]any{
            "severity": "SEV-2",
            "service":  "payments-api",
            "region":   "us-east-1",
        },
    }

    // Create CloudEvent wrapper
    ce := &models.CloudEvent{
        SpecVersion: "1.0",
        ID:          uuid.New().String(),
        Source:      "im://api",
        Type:        models.EventTypeDeclare,
        Subject:     fmt.Sprintf("incident/%s", incidentID),
        Time:        event.EventTime,
        Data:        event.Data,
        Extensions: map[string]any{
            models.ExtIncidentID: incidentID,
            models.ExtActor:      event.Actor,
        },
    }

    // Append to timeline
    ctx := context.Background()
    err := svc.AppendEvent(ctx, event, ce)
    if err != nil {
        return fmt.Errorf("failed to append event: %w", err)
    }

    return nil
}

Idempotent Event Append

Use im.origin_id extension for idempotency:

func appendAlertEvent(svc *timeline.Service, incidentID, alertID string) error {
    event := &models.TimelineEvent{
        IncidentID: incidentID,
        EventType:  models.EventTypeAlertTrigger,
        Actor:      "system:pagerduty",
        Title:      "High CPU Usage Alert",
        EventTime:  time.Now(),
        Source:     "pagerduty",
        Data: map[string]any{
            "alert_id": alertID,
            "metric":   "cpu.usage",
            "value":    95.2,
        },
    }

    ce := &models.CloudEvent{
        SpecVersion: "1.0",
        ID:          uuid.New().String(),
        Source:      "im://pagerduty",
        Type:        models.EventTypeAlertTrigger,
        Time:        event.EventTime,
        Extensions: map[string]any{
            models.ExtIncidentID: incidentID,
            models.ExtOriginID:   fmt.Sprintf("pagerduty:alert:%s", alertID),
            models.ExtActor:      event.Actor,
        },
    }

    ctx := context.Background()
    err := svc.AppendEvent(ctx, event, ce)

    // Note: Duplicate events (same origin_id) return nil error
    // but do not create a new entry
    return err
}

Event with Correlation Key

Group related events across incidents:

func appendCorrelatedEvent(svc *timeline.Service, incidentID, deploymentID string) error {
    ce := &models.CloudEvent{
        SpecVersion: "1.0",
        ID:          uuid.New().String(),
        Source:      "im://ci-cd",
        Type:        "im.deployment.complete",
        Time:        time.Now(),
        Extensions: map[string]any{
            models.ExtIncidentID:     incidentID,
            models.ExtCorrelationKey: fmt.Sprintf("deployment:%s", deploymentID),
            models.ExtActor:          "system:github-actions",
        },
    }

    event := &models.TimelineEvent{
        IncidentID: incidentID,
        EventType:  "im.deployment.complete",
        Actor:      "system:github-actions",
        Title:      "Hotfix Deployment Completed",
        EventTime:  ce.Time,
        Source:     "ci-cd",
    }

    ctx := context.Background()
    return svc.AppendEvent(ctx, event, ce)
}

Event with Sensitive Data

Mark events containing PII/PHI for automatic redaction:

func appendCustomerImpactEvent(svc *timeline.Service, incidentID string) error {
    event := &models.TimelineEvent{
        IncidentID: incidentID,
        EventType:  "im.incident.customer_impact",
        Actor:      "support@example.com",
        Title:      "Customer Reported Issue",
        EventTime:  time.Now(),
        Source:     "support-portal",
        Data: map[string]any{
            "customer_email": "customer@example.com",  // Will be redacted
            "customer_name":  "Jane Doe",              // Will be redacted
            "ticket_id":      "SUP-5678",              // Not redacted
        },
    }

    ce := &models.CloudEvent{
        SpecVersion: "1.0",
        ID:          uuid.New().String(),
        Source:      "im://support",
        Type:        "im.incident.customer_impact",
        Time:        event.EventTime,
        Extensions: map[string]any{
            models.ExtIncidentID: incidentID,
            models.ExtDataClass:  []string{"pii"},  // Mark as PII
            models.ExtVisibility: "restricted",      // Restrict access
            models.ExtActor:      event.Actor,
        },
    }

    ctx := context.Background()
    return svc.AppendEvent(ctx, event, ce)
}

Querying Events

Simple Timeline Query 

func getIncidentTimeline(svc *timeline.Service, incidentID string) ([]*models.TimelineEvent, error) {
    ctx := context.Background()

    // Get last 100 events
    events, err := svc.GetTimeline(ctx, incidentID, 100)
    if err != nil {
        return nil, fmt.Errorf("failed to get timeline: %w", err)
    }

    return events, nil
}

Advanced Query with Filters 

func queryFilteredTimeline(svc *timeline.Service, incidentID string) (*models.TimelineQueryResult, error) {
    // Parse time range
    from, _ := time.Parse(time.RFC3339, "2025-12-10T00:00:00Z")
    to, _ := time.Parse(time.RFC3339, "2025-12-10T23:59:59Z")

    query := &models.TimelineQuery{
        IncidentID:       incidentID,
        EventTypePattern: "im.incident.*",  // Only incident events
        FromTime:         &from,
        ToTime:           &to,
        Actor:            "alice@example.com",
        Limit:            50,
        Order:            "desc",  // Newest first
    }

    ctx := context.Background()
    result, err := svc.QueryTimeline(ctx, query)
    if err != nil {
        return nil, fmt.Errorf("failed to query timeline: %w", err)
    }

    return result, nil
}

Paginated Query 

func paginatedQuery(svc *timeline.Service, incidentID string) error {
    query := &models.TimelineQuery{
        IncidentID: incidentID,
        Limit:      50,
        Order:      "asc",  // Oldest first
    }

    ctx := context.Background()

    // Fetch all pages
    for {
        result, err := svc.QueryTimeline(ctx, query)
        if err != nil {
            return err
        }

        // Process events
        for _, event := range result.Events {
            fmt.Printf("[%s] %s: %s\n",
                event.EventTime.Format(time.RFC3339),
                event.EventType,
                event.Title)
        }

        // Check for more results
        if result.NextCursor == "" {
            break
        }

        // Update cursor for next page
        query.Cursor = result.NextCursor
    }

    return nil
}

Cross-Incident Query by Correlation Key 

func queryByCorrelationKey(svc *timeline.Service, correlationKey string) ([]*models.TimelineEvent, error) {
    query := &models.TimelineQuery{
        CorrelationKey: correlationKey,
        Limit:          100,
        Order:          "asc",
    }

    ctx := context.Background()
    result, err := svc.QueryTimeline(ctx, query)
    if err != nil {
        return nil, err
    }

    return result.Events, nil
}

Event Type Definitions

Standard Event Types 

package models

const (
    // Incident lifecycle
    EventTypeDeclare      = "im.incident.declare"
    EventTypeAcknowledge  = "im.incident.acknowledge"
    EventTypeEscalate     = "im.incident.escalate"
    EventTypeResolve      = "im.incident.resolve"
    EventTypeClose        = "im.incident.close"
    EventTypeReopen       = "im.incident.reopen"

    // Alerts
    EventTypeAlertTrigger = "im.alert.trigger"
    EventTypeAlertAck     = "im.alert.acknowledge"
    EventTypeAlertResolve = "im.alert.resolve"

    // Collaboration
    EventTypeNoteAdd      = "im.note.add"
    EventTypeStatusUpdate = "im.status_update.post"
    EventTypeRunbook      = "im.runbook.execute"

    // Integration sync
    EventTypeServiceNowSync = "im.servicenow.sync"
    EventTypeJiraSync       = "im.jira.sync"
    EventTypeSlackMessage   = "im.slack.message"
)

Custom Event Types

Define custom event types following the im.* namespace:

const (
    // Custom deployment events
    EventTypeDeploymentStart    = "im.deployment.start"
    EventTypeDeploymentComplete = "im.deployment.complete"
    EventTypeDeploymentRollback = "im.deployment.rollback"

    // Custom monitoring events
    EventTypeThresholdBreach = "im.monitor.threshold_breach"
    EventTypeAnomalyDetected = "im.monitor.anomaly_detected"
)

CloudEvents Extensions

Standard Extensions 

const (
    // Required for timeline storage
    ExtIncidentID     = "im.incident_id"      // Links to incident
    ExtActor          = "im.actor"            // Who/what triggered

    // Idempotency and grouping
    ExtOriginID       = "im.origin_id"        // Dedupe key from source
    ExtCorrelationKey = "im.correlation_key"  // Cross-event grouping
    ExtFingerprint    = "im.fingerprint"      // Auto-aggregation

    // Related events
    ExtRelatedIDs     = "im.related_ids"      // Parent/child refs

    // Security and access control
    ExtDataClass      = "im.data_class"       // ["pii", "phi", "financial"]
    ExtVisibility     = "im.visibility"       // "public", "restricted"
    ExtRedactionApplied = "im.redaction_applied" // Set by PEP

    // Metadata
    ExtTags           = "im.tags"             // Categorical tags
)

Extension Usage Examples 

// Example 1: Basic event
extensions := map[string]any{
    models.ExtIncidentID: "INC-1234",
    models.ExtActor:      "alice@example.com",
}

// Example 2: Idempotent event from external source
extensions := map[string]any{
    models.ExtIncidentID: "INC-1234",
    models.ExtOriginID:   "pagerduty:incident:ABCD123",
    models.ExtActor:      "system:pagerduty",
}

// Example 3: Correlated events
extensions := map[string]any{
    models.ExtIncidentID:     "INC-1234",
    models.ExtCorrelationKey: "deployment:v2.5.1",
    models.ExtActor:          "system:github-actions",
}

// Example 4: Sensitive data with access control
extensions := map[string]any{
    models.ExtIncidentID: "INC-1234",
    models.ExtDataClass:  []string{"pii", "financial"},
    models.ExtVisibility: "confidential",
    models.ExtActor:      "billing@example.com",
}

// Example 5: Related events (parent/child)
extensions := map[string]any{
    models.ExtIncidentID:  "INC-1235",
    models.ExtRelatedIDs:  []string{"INC-1234"},  // Parent incident
    models.ExtActor:       "alice@example.com",
}

Connector Integration Patterns

Pattern 1: Webhook Ingestor 

type PagerDutyWebhook struct {
    timelineSvc *timeline.Service
}

func (h *PagerDutyWebhook) HandleIncidentTrigger(w http.ResponseWriter, r *http.Request) {
    var payload PagerDutyPayload
    json.NewDecoder(r.Body).Decode(&payload)

    // Map to incident ID (or create new incident)
    incidentID := h.getOrCreateIncident(payload.IncidentKey)

    // Create timeline event
    event := &models.TimelineEvent{
        IncidentID: incidentID,
        EventType:  models.EventTypeAlertTrigger,
        Actor:      "system:pagerduty",
        Title:      payload.IncidentTitle,
        EventTime:  payload.CreatedAt,
        Source:     "pagerduty",
        Data:       payload,
    }

    ce := &models.CloudEvent{
        SpecVersion: "1.0",
        ID:          uuid.New().String(),
        Source:      "im://pagerduty",
        Type:        models.EventTypeAlertTrigger,
        Time:        payload.CreatedAt,
        Extensions: map[string]any{
            models.ExtOriginID:   fmt.Sprintf("pd:%s", payload.IncidentID),
            models.ExtIncidentID: incidentID,
            models.ExtActor:      "system:pagerduty",
        },
    }

    // Append (idempotent - duplicate webhooks ignored)
    err := h.timelineSvc.AppendEvent(r.Context(), event, ce)
    if err != nil {
        http.Error(w, "Failed to append event", http.StatusInternalServerError)
        return
    }

    w.WriteHeader(http.StatusOK)
}

Pattern 2: Poll-Based Sync 

func syncServiceNowTickets(timelineSvc *timeline.Service, snowClient *ServiceNowClient) error {
    // Query ServiceNow for updated tickets
    since := time.Now().Add(-5 * time.Minute)
    tickets, err := snowClient.GetUpdatedTickets(since)
    if err != nil {
        return err
    }

    // Append timeline event for each update
    for _, ticket := range tickets {
        incidentID := mapTicketToIncident(ticket.Number)

        event := &models.TimelineEvent{
            IncidentID: incidentID,
            EventType:  "im.servicenow.sync",
            Actor:      fmt.Sprintf("system:servicenow:%s", ticket.UpdatedBy),
            Title:      ticket.ShortDescription,
            EventTime:  ticket.UpdatedAt,
            Source:     "servicenow",
            Data:       ticket,
        }

        ce := &models.CloudEvent{
            SpecVersion: "1.0",
            ID:          uuid.New().String(),
            Source:      "im://servicenow",
            Type:        "im.servicenow.sync",
            Time:        ticket.UpdatedAt,
            Extensions: map[string]any{
                models.ExtOriginID:   fmt.Sprintf("snow:%s:%s", ticket.SysID, ticket.SysModCount),
                models.ExtIncidentID: incidentID,
            },
        }

        // Idempotent append - same sys_mod_count ignored
        if err := timelineSvc.AppendEvent(context.Background(), event, ce); err != nil {
            log.Printf("Failed to sync ticket %s: %v", ticket.Number, err)
        }
    }

    return nil
}

Pattern 3: Event Bus Subscriber 

func subscribeToIncidentEvents(timelineSvc *timeline.Service, eventBus *EventBus) {
    // Subscribe to incident notifications
    eventBus.Subscribe("im.incident.*", func(notification Event) {
        // Extract event details
        incidentID := notification.Subject

        event := &models.TimelineEvent{
            IncidentID: incidentID,
            EventType:  notification.Type,
            Actor:      notification.GetExtension(models.ExtActor).(string),
            Title:      notification.Summary,
            EventTime:  notification.Time,
            Source:     "event-bus",
            Data:       notification.Data,
        }

        ce := &models.CloudEvent{
            SpecVersion: "1.0",
            ID:          notification.ID,
            Source:      notification.Source,
            Type:        notification.Type,
            Time:        notification.Time,
            Extensions:  notification.Extensions,
        }

        // Append to timeline
        ctx := context.Background()
        if err := timelineSvc.AppendEvent(ctx, event, ce); err != nil {
            log.Printf("Failed to append event: %v", err)
        }
    })
}

Testing

Unit Test Example 

func TestAppendEvent_Idempotency(t *testing.T) {
    // Setup
    db := storage.NewTestDB(t)
    svc := timeline.New(db)

    incidentID := "INC-TEST-1234"
    originID := "test:event:1"

    event := &models.TimelineEvent{
        IncidentID: incidentID,
        EventType:  models.EventTypeDeclare,
        Actor:      "test@example.com",
        Title:      "Test Event",
        EventTime:  time.Now(),
        Source:     "test",
    }

    ce := &models.CloudEvent{
        SpecVersion: "1.0",
        ID:          uuid.New().String(),
        Source:      "im://test",
        Type:        models.EventTypeDeclare,
        Time:        event.EventTime,
        Extensions: map[string]any{
            models.ExtOriginID:   originID,
            models.ExtIncidentID: incidentID,
            models.ExtActor:      event.Actor,
        },
    }

    ctx := context.Background()

    // First append
    err := svc.AppendEvent(ctx, event, ce)
    require.NoError(t, err)

    // Get timeline
    events1, err := svc.GetTimeline(ctx, incidentID, 100)
    require.NoError(t, err)
    require.Len(t, events1, 1)
    firstEventID := events1[0].ID

    // Second append with same origin_id
    err = svc.AppendEvent(ctx, event, ce)
    require.NoError(t, err)  // No error on duplicate

    // Verify no duplicate created
    events2, err := svc.GetTimeline(ctx, incidentID, 100)
    require.NoError(t, err)
    require.Len(t, events2, 1)
    require.Equal(t, firstEventID, events2[0].ID)
}

Best Practices

1. Always Use Origin IDs for External Events 

// Good: Idempotent
extensions := map[string]any{
    models.ExtOriginID: "pagerduty:alert:ABC123",
}

// Bad: Duplicate events on webhook retries
extensions := map[string]any{
    // No origin_id - relies only on UUID
}

2. Set Appropriate Visibility Levels 

// Public event (default)
extensions := map[string]any{
    models.ExtVisibility: "public",
}

// Restricted event (requires attribute)
extensions := map[string]any{
    models.ExtVisibility: "restricted",
}

// Confidential event
extensions := map[string]any{
    models.ExtVisibility: "confidential",
}

3. Tag Sensitive Data 

// Mark PII for automatic redaction
extensions := map[string]any{
    models.ExtDataClass: []string{"pii"},
}

// Mark multiple classifications
extensions := map[string]any{
    models.ExtDataClass: []string{"pii", "financial"},
}
// Group events from same deployment
extensions := map[string]any{
    models.ExtCorrelationKey: "deployment:v2.5.1",
}

// Query all related events later
query := &models.TimelineQuery{
    CorrelationKey: "deployment:v2.5.1",
}

5. Handle Errors Gracefully 

func appendEventWithRetry(svc *timeline.Service, event *models.TimelineEvent, ce *models.CloudEvent) error {
    ctx := context.Background()

    // Retry with exponential backoff
    return retry.Do(
        func() error {
            return svc.AppendEvent(ctx, event, ce)
        },
        retry.Attempts(3),
        retry.Delay(100 * time.Millisecond),
    )
}

Performance Considerations

  1. Batch Inserts: For high-throughput scenarios, batch multiple events in a transaction
  2. Async Processing: Queue events and process asynchronously for non-critical paths
  3. Connection Pooling: Use database connection pooling for concurrent writes
  4. Read Replicas: Query timelines from read replicas to reduce primary load

Next Steps

Edit this page on GitHub
Prev
Event Distribution Guide