Spaces:

Leon4gr45
/

openmeter

Paused

openmeter / sink /flushhandler /ingestnotification /handler.go

Upload folder using huggingface_hub (part 6)

d6f631f verified 4 months ago

4.55 kB

package ingestnotification

import (
	"context"
	"errors"
	"log/slog"
	"slices"
	"time"

	"github.com/samber/lo"
	"go.opentelemetry.io/otel/metric"

	eventmodels "github.com/openmeterio/openmeter/openmeter/event/models"
	"github.com/openmeterio/openmeter/openmeter/ingest/kafkaingest/serializer"
	"github.com/openmeterio/openmeter/openmeter/meter"
	"github.com/openmeterio/openmeter/openmeter/sink/flushhandler"
	ingestevents "github.com/openmeterio/openmeter/openmeter/sink/flushhandler/ingestnotification/events"
	sinkmodels "github.com/openmeterio/openmeter/openmeter/sink/models"
	"github.com/openmeterio/openmeter/openmeter/watermill/eventbus"
	"github.com/openmeterio/openmeter/pkg/slicesx"
)

type handler struct {
	publisher eventbus.Publisher
	logger    *slog.Logger
	config    HandlerConfig
}

type HandlerConfig struct {
	MaxEventsInBatch int
}

func (c HandlerConfig) Validate() error {
	if c.MaxEventsInBatch <= 0 {
		return errors.New("max_events_in_batch must be greater than 0")
	}

	return nil
}

func NewHandler(logger *slog.Logger, metricMeter metric.Meter, publisher eventbus.Publisher, config HandlerConfig) (flushhandler.FlushEventHandler, error) {
	if err := config.Validate(); err != nil {
		return nil, err
	}

	handler := &handler{
		publisher: publisher,
		logger:    logger,
		config:    config,
	}

	return flushhandler.NewFlushEventHandler(
		flushhandler.FlushEventHandlerOptions{
			Name:        "ingest_notification",
			Callback:    handler.OnFlushSuccess,
			Logger:      logger,
			MetricMeter: metricMeter,
		})
}

// OnFlushSuccess takes a look at the incoming messages and in case something is
// affecting a ledger balance it will create the relevant event.
// We could resolve the customers in the event processing / generation instead of in the handlers. There are pros and cons to both.
func (h *handler) OnFlushSuccess(ctx context.Context, events []sinkmodels.SinkMessage) error {
	var finalErr error

	// Filter meaningful events for downstream
	filtered := lo.Filter(events, func(event sinkmodels.SinkMessage, _ int) bool {
		return event.Serialized != nil
	})

	if len(filtered) == 0 {
		h.logger.Debug("no events to process in batch for ingest notification")
		return nil
	}

	now := time.Now()

	// Map the filtered events to the ingest event
	iEvents := slicesx.Map(filtered, func(message sinkmodels.SinkMessage) ingestevents.EventBatchedIngest {
		res := ingestevents.EventBatchedIngest{
			Namespace:  eventmodels.NamespaceID{ID: message.Namespace},
			SubjectKey: message.Serialized.Subject,
			MeterSlugs: h.getMeterSlugsFromMeters(message.Meters),
			// Warning: Given this is called after the clickhouse writes have completed, it's a fair assumption that
			// the event was stored at this time to clickhouse.
			StoredAt: now,
		}

		if message.Serialized != nil {
			res.RawEvents = append(res.RawEvents, *message.Serialized)
		}

		return res
	})

	// Let's group the events by subject
	iEventsBySubject := lo.GroupBy(iEvents, func(event ingestevents.EventBatchedIngest) string {
		return event.Namespace.ID + "/" + event.SubjectKey
	})

	// Let's merge the events by subject
	iEvents = make([]ingestevents.EventBatchedIngest, 0, len(iEventsBySubject))
	for _, events := range iEventsBySubject {
		if len(events) == 0 {
			continue
		}

		if len(events) == 1 {
			iEvents = append(iEvents, events[0])
			continue
		}

		chunkedEvents := lo.Chunk(events, h.config.MaxEventsInBatch)

		for _, chunk := range chunkedEvents {
			event := ingestevents.EventBatchedIngest{
				Namespace:  chunk[0].Namespace,
				SubjectKey: chunk[0].SubjectKey,
				StoredAt:   now,
			}

			event.MeterSlugs = lo.Uniq(
				slices.Concat(
					lo.Map(chunk, func(event ingestevents.EventBatchedIngest, _ int) []string {
						return event.MeterSlugs
					})...,
				),
			)

			event.RawEvents = slices.Concat(
				lo.Map(chunk, func(event ingestevents.EventBatchedIngest, _ int) []serializer.CloudEventsKafkaPayload {
					return event.RawEvents
				})...,
			)

			iEvents = append(iEvents, event)
		}
	}

	// We need to chunk the events to not exceed message size limits
	for _, event := range iEvents {
		if err := h.publisher.Publish(ctx, event); err != nil {
			finalErr = errors.Join(finalErr, err)
			h.logger.ErrorContext(ctx, "failed to publish change notification", "error", err)
		}
	}

	return finalErr
}

func (h *handler) getMeterSlugsFromMeters(meters []*meter.Meter) []string {
	slugs := make([]string, len(meters))
	for i, meter := range meters {
		slugs[i] = meter.Key
	}

	return slugs
}

	package ingestnotification

	import (
	"context"
	"errors"
	"log/slog"
	"slices"
	"time"

	"github.com/samber/lo"
	"go.opentelemetry.io/otel/metric"

	eventmodels "github.com/openmeterio/openmeter/openmeter/event/models"
	"github.com/openmeterio/openmeter/openmeter/ingest/kafkaingest/serializer"
	"github.com/openmeterio/openmeter/openmeter/meter"
	"github.com/openmeterio/openmeter/openmeter/sink/flushhandler"
	ingestevents "github.com/openmeterio/openmeter/openmeter/sink/flushhandler/ingestnotification/events"
	sinkmodels "github.com/openmeterio/openmeter/openmeter/sink/models"
	"github.com/openmeterio/openmeter/openmeter/watermill/eventbus"
	"github.com/openmeterio/openmeter/pkg/slicesx"
	)

	type handler struct {
	publisher eventbus.Publisher
	logger *slog.Logger
	config HandlerConfig
	}

	type HandlerConfig struct {
	MaxEventsInBatch int
	}

	func (c HandlerConfig) Validate() error {
	if c.MaxEventsInBatch <= 0 {
	return errors.New("max_events_in_batch must be greater than 0")
	}

	return nil
	}

	func NewHandler(logger *slog.Logger, metricMeter metric.Meter, publisher eventbus.Publisher, config HandlerConfig) (flushhandler.FlushEventHandler, error) {
	if err := config.Validate(); err != nil {
	return nil, err
	}

	handler := &handler{
	publisher: publisher,
	logger: logger,
	config: config,
	}

	return flushhandler.NewFlushEventHandler(
	flushhandler.FlushEventHandlerOptions{
	Name: "ingest_notification",
	Callback: handler.OnFlushSuccess,
	Logger: logger,
	MetricMeter: metricMeter,
	})
	}

	// OnFlushSuccess takes a look at the incoming messages and in case something is
	// affecting a ledger balance it will create the relevant event.
	// We could resolve the customers in the event processing / generation instead of in the handlers. There are pros and cons to both.
	func (h *handler) OnFlushSuccess(ctx context.Context, events []sinkmodels.SinkMessage) error {
	var finalErr error

	// Filter meaningful events for downstream
	filtered := lo.Filter(events, func(event sinkmodels.SinkMessage, _ int) bool {
	return event.Serialized != nil
	})

	if len(filtered) == 0 {
	h.logger.Debug("no events to process in batch for ingest notification")
	return nil
	}

	now := time.Now()

	// Map the filtered events to the ingest event
	iEvents := slicesx.Map(filtered, func(message sinkmodels.SinkMessage) ingestevents.EventBatchedIngest {
	res := ingestevents.EventBatchedIngest{
	Namespace: eventmodels.NamespaceID{ID: message.Namespace},
	SubjectKey: message.Serialized.Subject,
	MeterSlugs: h.getMeterSlugsFromMeters(message.Meters),
	// Warning: Given this is called after the clickhouse writes have completed, it's a fair assumption that
	// the event was stored at this time to clickhouse.
	StoredAt: now,
	}

	if message.Serialized != nil {
	res.RawEvents = append(res.RawEvents, *message.Serialized)
	}

	return res
	})

	// Let's group the events by subject
	iEventsBySubject := lo.GroupBy(iEvents, func(event ingestevents.EventBatchedIngest) string {
	return event.Namespace.ID + "/" + event.SubjectKey
	})

	// Let's merge the events by subject
	iEvents = make([]ingestevents.EventBatchedIngest, 0, len(iEventsBySubject))
	for _, events := range iEventsBySubject {
	if len(events) == 0 {
	continue
	}

	if len(events) == 1 {
	iEvents = append(iEvents, events[0])
	continue
	}

	chunkedEvents := lo.Chunk(events, h.config.MaxEventsInBatch)

	for _, chunk := range chunkedEvents {
	event := ingestevents.EventBatchedIngest{
	Namespace: chunk[0].Namespace,
	SubjectKey: chunk[0].SubjectKey,
	StoredAt: now,
	}

	event.MeterSlugs = lo.Uniq(
	slices.Concat(
	lo.Map(chunk, func(event ingestevents.EventBatchedIngest, _ int) []string {
	return event.MeterSlugs
	})...,
	),
	)

	event.RawEvents = slices.Concat(
	lo.Map(chunk, func(event ingestevents.EventBatchedIngest, _ int) []serializer.CloudEventsKafkaPayload {
	return event.RawEvents
	})...,
	)

	iEvents = append(iEvents, event)
	}
	}

	// We need to chunk the events to not exceed message size limits
	for _, event := range iEvents {
	if err := h.publisher.Publish(ctx, event); err != nil {
	finalErr = errors.Join(finalErr, err)
	h.logger.ErrorContext(ctx, "failed to publish change notification", "error", err)
	}
	}

	return finalErr
	}

	func (h handler) getMeterSlugsFromMeters(meters []meter.Meter) []string {
	slugs := make([]string, len(meters))
	for i, meter := range meters {
	slugs[i] = meter.Key
	}

	return slugs
	}