import json
import boto3
from kafka import KafkaConsumer
from datetime import datetime

# Setup MinIO connection
s3 = boto3.client(
    's3',
    endpoint_url='http://minio:9000',
    aws_access_key_id='admin',
    aws_secret_access_key='admin123456'
)

# Setup Kafka Consumer
consumer = KafkaConsumer(
    'network-logs',
    bootstrap_servers=['localhost:9092'],
    value_deserializer=lambda x: json.loads(x.decode('utf-8')),
    auto_offset_reset='earliest',
    group_id='minio-consumer'
)

print("Listening for logs from Kafka...")

batch = []
batch_size = 20  # Save every 20 logs as one file

for message in consumer:
    log = message.value
    batch.append(json.dumps(log))

    if len(batch) >= batch_size:
        # Generate filename based on current timestamp
        filename = f"logs/{datetime.now().strftime('%Y/%m/%d/%H-%M-%S')}.json"
        content  = "\n".join(batch)

        s3.put_object(
            Bucket='raw-logs',
            Key=filename,
            Body=content.encode('utf-8')
        )

        print(f"Saved {len(batch)} logs to MinIO: {filename}")
        batch = []
