shadow_compose.yml

# TAP project 'The Observer' a.a. 2022/2023 {Potentially defective compose ~ for debug purposes}
version: '3.3'
services:

  # datasource ==> mastodon.uno
  stream_connector_1:
    image: tap_project:streamConnector
    container_name: streamConnector_1
    networks:
      - tap_project_2
    environment:
      - STREAM_ENDPOINT=https://mastodon.uno/api/v1/streaming/public
      - STREAM_API_NAME=mastodonuno
    volumes:
      - datastorage:/app/data
    restart: unless-stopped
    build:
      context: ./Stream_Connector/
      dockerfile: Dockerfile

  # datasource ==> mas.to
  stream_connector_2:
    image: tap_project:streamConnector
    container_name: streamConnector_2
    networks:
      - tap_project_2
    environment:
      - STREAM_ENDPOINT=https://mas.to/api/v1/streaming/public
      - STREAM_API_NAME=masto
    volumes:
      - datastorage:/app/data
    restart: unless-stopped
    build:
      context: ./Stream_Connector/
      dockerfile: Dockerfile

  # datasource ==> mstdn.social
  stream_connector_3:
    image: tap_project:streamConnector
    container_name: streamConnector_3
    networks:
      - tap_project_2
    environment:
      - STREAM_ENDPOINT=https://mstdn.social/api/v1/streaming/public
      - STREAM_API_NAME=mstdnsocial
    volumes:
      - datastorage:/app/data
    restart: unless-stopped
    build:
      context: ./Stream_Connector/
      dockerfile: Dockerfile

  # datasource ==> fosstodon.org
  stream_connector_4:
    image: tap_project:streamConnector
    container_name: streamConnector_4
    networks:
      - tap_project_2
    environment:
      - STREAM_ENDPOINT=https://fosstodon.org/api/v1/streaming/public
      - STREAM_API_NAME=fosstodonorg
    volumes:
      - datastorage:/app/data
    restart: unless-stopped
    build:
      context: ./Stream_Connector/
      dockerfile: Dockerfile

  # datasource ==> climatejustice.social
  stream_connector_5:
    image: tap_project:streamConnector
    container_name: streamConnector_5
    networks:
      - tap_project_2
    environment:
      - STREAM_ENDPOINT=https://climatejustice.social/api/v1/streaming/public
      - STREAM_API_NAME=climatejusticesocial
    volumes:
      - datastorage:/app/data
    restart: unless-stopped
    build:
      context: ./Stream_Connector/
      dockerfile: Dockerfile

  # datasource ==> ravenation.club
  stream_connector_6:
    image: tap_project:streamConnector
    container_name: streamConnector_6
    networks:
      - tap_project_2
    environment:
      - STREAM_ENDPOINT=https://ravenation.club/api/v1/streaming/public
      - STREAM_API_NAME=ravenationclub
    volumes:
      - datastorage:/app/data
    restart: unless-stopped
    build:
      context: ./Stream_Connector/
      dockerfile: Dockerfile

  # datasource ==> hostux.social
  stream_connector_7:
    image: tap_project:streamConnector
    container_name: streamConnector_7
    networks:
      - tap_project_2
    environment:
      - STREAM_ENDPOINT=https://hostux.social/api/v1/streaming/public
      - STREAM_API_NAME=hostuxsocial
    volumes:
      - datastorage:/app/data
    restart: unless-stopped
    build:
      context: ./Stream_Connector/
      dockerfile: Dockerfile

  # server ~ mastodonuno
  data_ingestor_1:
    image: tap_project:data_ingestor
    container_name: dataIngestor_1
    environment:
      - DATA_SOURCE=mastodonuno
    networks:
      - tap_project_2
    volumes:
      - datastorage:/data
    restart: unless-stopped
    build:
      context: ./Data_Ingestion/containerized_camel/
      dockerfile: Dockerfile

  # server ~ mas.to
  data_ingestor_2:
    image: tap_project:data_ingestor
    container_name: dataIngestor_2
    environment:
      - DATA_SOURCE=masto
    networks:
      - tap_project_2
    volumes:
      - datastorage:/data
    restart: unless-stopped
    build:
      context: ./Data_Ingestion/containerized_camel/
      dockerfile: Dockerfile

  # server ~ mstdn.social
  data_ingestor_3:
    image: tap_project:data_ingestor
    container_name: dataIngestor_3
    environment:
      - DATA_SOURCE=mstdnsocial
    networks:
      - tap_project_2
    volumes:
      - datastorage:/data
    restart: unless-stopped
    build:
      context: ./Data_Ingestion/containerized_camel/
      dockerfile: Dockerfile

  # server ~ fosstodon.org
  data_ingestor_4:
    image: tap_project:data_ingestor
    container_name: dataIngestor_4
    environment:
      - DATA_SOURCE=fosstodonorg
    networks:
      - tap_project_2
    volumes:
      - datastorage:/data
    restart: unless-stopped
    build:
      context: ./Data_Ingestion/containerized_camel/
      dockerfile: Dockerfile

  # server ~ climatejustice.social
  data_ingestor_5:
    image: tap_project:data_ingestor
    container_name: dataIngestor_5
    environment:
      - DATA_SOURCE=climatejusticesocial
    networks:
      - tap_project_2
    volumes:
      - datastorage:/data
    restart: unless-stopped
    build:
      context: ./Data_Ingestion/containerized_camel/
      dockerfile: Dockerfile
  
  # server ~ revenation.club
  data_ingestor_6:
    image: tap_project:data_ingestor
    container_name: dataIngestor_6
    environment:
      - DATA_SOURCE=ravenationclub
    networks:
      - tap_project_2
    volumes:
      - datastorage:/data
    restart: unless-stopped
    build:
      context: ./Data_Ingestion/containerized_camel/
      dockerfile: Dockerfile

  # server ~ hostux.social
  data_ingestor_7:
    image: tap_project:data_ingestor
    container_name: dataIngestor_7
    environment:
      - DATA_SOURCE=hostuxsocial
    networks:
      - tap_project_2
    volumes:
      - datastorage:/data
    restart: unless-stopped
    build:
      context: ./Data_Ingestion/containerized_camel/
      dockerfile: Dockerfile

  data_streamer_zookeeper:
    image: tap_project:kafka
    container_name: kafkaZK
    environment:
      - KAFKA_ACTION=start-zk
    networks:
      tap_project_2:
        ipv4_address: 10.0.100.22
    ports:
      - "2181:2181"
    restart: unless-stopped
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile


# Turn on only for debugging
  data_streamer_zookeeper_ui:
    image: juris/zkui
    container_name: zkui
    environment:
      - ZK_SERVER=10.0.100.22:2181
    # Debug here
    ports:
      - "9096:9090"
    networks:
      - tap_project_2
    depends_on:
      - data_streamer_zookeeper
    restart: unless-stopped
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  data_streamer_kafkaServer:
    hostname: kafkaServer
    image: tap_project:kafka
    container_name: kafkaServer
    environment:
      - KAFKA_ACTION=start-kafka
    networks:
      - tap_project_2
        # ipv4_address: 10.0.100.23
    ports:
      - "9092:9092"
    depends_on:
      - data_streamer_zookeeper
    restart: unless-stopped
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  data_streamer_kafkaServer_ui:
    image: provectuslabs/kafka-ui:v0.7.0
    # image: provectuslabs/kafka-ui:master
    container_name: kafkaUi
    environment:
      - KAFKA_CLUSTERS_0_NAME=local
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafkaServer:9092
    networks:
      - tap_project_2
    # Debugging
    ports:
      - "8090:8080"
    depends_on:
      - data_streamer_kafkaServer
    restart: unless-stopped
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  # topic --> mastodonuno
  topic_create_1:
    image: tap_project:kafka
    environment:
      - KAFKA_ACTION=create-topic
      - KAKFA_SERVER=10.0.100.23
      - KAFKA_TOPIC=mastodonuno
    networks:
      tap_project_2:
        ipv4_address: 10.0.100.24
    depends_on:
      - data_streamer_kafkaServer
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  # topic --> masto
  topic_create_2:
    image: tap_project:kafka
    environment:
      - KAFKA_ACTION=create-topic
      - KAKFA_SERVER=10.0.100.23
      - KAFKA_TOPIC=masto
    networks:
      tap_project_2:
        ipv4_address: 10.0.100.25
    depends_on:
      - data_streamer_kafkaServer
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  # topic --> mstdnsocial
  topic_create_3:
    image: tap_project:kafka
    environment:
      - KAFKA_ACTION=create-topic
      - KAKFA_SERVER=10.0.100.23
      - KAFKA_TOPIC=mstdnsocial
    networks:
      tap_project_2:
        ipv4_address: 10.0.100.26
    depends_on:
      - data_streamer_kafkaServer
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  # topic --> fosstodon.orgs
  topic_create_4:
    image: tap_project:kafka
    environment:
      - KAFKA_ACTION=create-topic
      - KAKFA_SERVER=10.0.100.23
      - KAFKA_TOPIC=fosstodonorg
    networks:
      tap_project_2:
        ipv4_address: 10.0.100.27
    depends_on:
      - data_streamer_kafkaServer
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  # topic --> climatejusticesocial
  topic_create_5:
    image: tap_project:kafka
    environment:
      - KAFKA_ACTION=create-topic
      - KAKFA_SERVER=10.0.100.23
      - KAFKA_TOPIC=climatejusticesocial
    networks:
      tap_project_2:
        ipv4_address: 10.0.100.28
    depends_on:
      - data_streamer_kafkaServer
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  # topic --> ravenationclub
  topic_create_6:
    image: tap_project:kafka
    environment:
      - KAFKA_ACTION=create-topic
      - KAKFA_SERVER=10.0.100.23
      - KAFKA_TOPIC=ravenationclub
    networks:
      tap_project_2:
        ipv4_address: 10.0.100.29
    depends_on:
      - data_streamer_kafkaServer
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  # topic --> hostuxsocial
  topic_create_7:
    image: tap_project:kafka
    environment:
      - KAFKA_ACTION=create-topic
      - KAKFA_SERVER=10.0.100.23
      - KAFKA_TOPIC=hostuxsocial
    networks:
      tap_project_2:
        ipv4_address: 10.0.100.30
    depends_on:
      - data_streamer_kafkaServer
    build:
      context: ./Data_Streaming
      dockerfile: Dockerfile

  # stream ~ mastodonuno
  spark_1:
      image: tap_project:spark
      container_name: spark_1
      environment:
        - SPARK_ACTION=bash
        - SPARK_MASTER_URL=spark://spark-master:7077
        - SPARK_DEPLOY_MODE=client
        - SPARK_APPLICATION_PYTHON_FILES=/opt/spark-app/topic_modelling.py
        - KAFKA_TOPIC=mastodonuno
      # Remember the topic ^
        - ELASTIC_HOST_URL=http://elasticsearch:9200
      # Configure the Data Indexer dynamically
        - KAFKA_SERVER_URL=kafkaServer:9092
      # Configure the kafka server dynamically
        - ELASTIC_SERVER_HOSTNAME=elasticsearch
      # Configure the elasticsearch hostname
      volumes:
        - ./Data_Processing/code:/opt/spark-app
        - ./Data_Processing/models:/opt/spark
      deploy:
        # Tweak with caution
        resources:
          limits:
            cpus: '3.0'
            memory: 4g
      networks:
        - tap_project_2
          #ipv4_address: 10.0.100.2
      command: spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.2,org.elasticsearch:elasticsearch-spark-30_2.12:8.2.0 --master local[*] /opt/spark-app/topic_modelling.py
      depends_on:
        - data_streamer_kafkaServer
      restart: unless-stopped
      build:
        context: ./Data_Processing
        dockerfile: Dockerfile

  # stream ~ masto
  spark_2:
      image: tap_project:spark
      container_name: spark_2
      environment:
        - SPARK_ACTION=bash
        - SPARK_MASTER_URL=spark://spark-master:7077
        - SPARK_DEPLOY_MODE=client
        - SPARK_APPLICATION_PYTHON_FILES=/opt/spark-app/topic_modelling.py
        - KAFKA_TOPIC=masto
      # Remember the topic ^
        - ELASTIC_HOST_URL=http://elasticsearch:9200
      # Configure the Data Indexer dynamically
        - KAFKA_SERVER_URL=kafkaServer:9092
      # Configure the kafka server dynamically
        - ELASTIC_SERVER_HOSTNAME=elasticsearch
      # Configure the elasticsearch hostname
      volumes:
        - ./Data_Processing/code:/opt/spark-app
        - ./Data_Processing/models:/opt/spark
      deploy:
        resources:
          limits:
            cpus: '3.0'
            memory: 4g
      networks:
        - tap_project_2
          #ipv4_address: 10.0.100.2
      command: spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.2,org.elasticsearch:elasticsearch-spark-30_2.12:8.2.0 --master local[*] /opt/spark-app/topic_modelling.py
      depends_on:
        - data_streamer_kafkaServer
      restart: unless-stopped
      build:
        context: ./Data_Processing
        dockerfile: Dockerfile

  # stream ~ mstdn social
  #spark_3:
  #    image: tap_project:spark
  #    container_name: spark_3
  #    environment:
  #      - SPARK_ACTION=bash
  #      - SPARK_MASTER_URL=spark://spark-master:7077
  #      - SPARK_DEPLOY_MODE=client
  #      - SPARK_APPLICATION_PYTHON_FILES=/opt/spark-app/topic_modelling.py
  #      - KAFKA_TOPIC=mstdnsocial
  #    # Remember the topic ^
  #      - ELASTIC_HOST_URL=http://elasticsearch:9200
  #    # Configure the Data Indexer dynamically
  #      - KAFKA_SERVER_URL=kafkaServer:9092
  #    # Configure the kafka server dynamically
  #      - ELASTIC_SERVER_HOSTNAME=elasticsearch
  #    # Configure the elasticsearch hostname
  #    volumes:
  #      - ./Data_Processing/code:/opt/spark-app
  #      - ./Data_Processing/models:/opt/spark
  #    deploy:
  #      resources:
  #        limits:
  #          cpus: '3.0'
  #          memory: 4g
  #    networks:
  #      - tap_project_2
  #        #ipv4_address: 10.0.100.2
  #    command: spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.2,org.elasticsearch:elasticsearch-spark-30_2.12:8.2.0 --master local[*] /opt/spark-app/topic_modelling.py
  #    depends_on:
  #      - data_streamer_kafkaServer
  #    restart: unless-stopped
  #    build:
  #      context: ./Data_Processing
  #      dockerfile: Dockerfile


  # stream ~ fosstodonorg
  #spark_4:
  #    image: tap_project:spark
  #    container_name: spark_4
  #    environment:
  #      - SPARK_ACTION=bash
  #      - SPARK_MASTER_URL=spark://spark-master:7077
  #      - SPARK_DEPLOY_MODE=client
  #      - SPARK_APPLICATION_PYTHON_FILES=/opt/spark-app/topic_modelling.py
  #      - KAFKA_TOPIC=fosstodonorg
  #    # Remember the topic ^
  #      - ELASTIC_HOST_URL=http://elasticsearch:9200
  #    # Configure the Data Indexer dynamically
  #      - KAFKA_SERVER_URL=kafkaServer:9092
  #    # Configure the kafka server dynamically
  #      - ELASTIC_SERVER_HOSTNAME=elasticsearch
  #    # Configure the elasticsearch hostname
  #    volumes:
  #      - ./Data_Processing/code:/opt/spark-app
  #      - ./Data_Processing/models:/opt/spark
  #    deploy:
  #      resources:
  #        limits:
  #          cpus: '3.0'
  #          memory: 4g
  #    networks:
  #      - tap_project_2
  #        #ipv4_address: 10.0.100.2
  #    command: spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.2,org.elasticsearch:elasticsearch-spark-30_2.12:8.2.0 --master local[*] /opt/spark-app/topic_modelling.py
  #    depends_on:
  #      - data_streamer_kafkaServer
  #    restart: unless-stopped
  #    build:
  #      context: ./Data_Processing
  #      dockerfile: Dockerfile

  # stream ~ climatejusticesocial
  #spark_5:
  #    image: tap_project:spark
  #    container_name: spark_5
  #    environment:
  #      - SPARK_ACTION=bash
  #      - SPARK_MASTER_URL=spark://spark-master:7077
  #      - SPARK_DEPLOY_MODE=client
  #      - SPARK_APPLICATION_PYTHON_FILES=/opt/spark-app/topic_modelling.py
  #      - KAFKA_TOPIC=climatejusticesocial
  #    # Remember the topic ^
  #      - ELASTIC_HOST_URL=http://elasticsearch:9200
  #    # Configure the Data Indexer dynamically
  #      - KAFKA_SERVER_URL=kafkaServer:9092
  #    # Configure the kafka server dynamically
  #      - ELASTIC_SERVER_HOSTNAME=elasticsearch
  #    # Configure the elasticsearch hostname
  #    volumes:
  #      - ./Data_Processing/code:/opt/spark-app
  #      - ./Data_Processing/models:/opt/spark
  #    deploy:
  #      resources:
  #        limits:
  #          cpus: '3.0'
  #          memory: 4g
  #    networks:
  #      - tap_project_2
  #        #ipv4_address: 10.0.100.2
  #    command: spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.2,org.elasticsearch:elasticsearch-spark-30_2.12:8.2.0 --master local[*] /opt/spark-app/topic_modelling.py
  #    depends_on:
  #      - data_streamer_kafkaServer
  #    restart: unless-stopped
  #    build:
  #      context: ./Data_Processing
  #      dockerfile: Dockerfile

  # stream ~ ravenationclub
  #spark_6:
  #    image: tap_project:spark
  #    container_name: spark_6
  #    environment:
  #      - SPARK_ACTION=bash
  #      - SPARK_MASTER_URL=spark://spark-master:7077
  #     - SPARK_DEPLOY_MODE=client
  #      - SPARK_APPLICATION_PYTHON_FILES=/opt/spark-app/topic_modelling.py
  #      - KAFKA_TOPIC=ravenationclub
  #    # Remember the topic ^
  #      - ELASTIC_HOST_URL=http://elasticsearch:9200
  #    # Configure the Data Indexer dynamically
  #      - KAFKA_SERVER_URL=kafkaServer:9092
  #    # Configure the kafka server dynamically
  #      - ELASTIC_SERVER_HOSTNAME=elasticsearch
  #    # Configure the elasticsearch hostname
  #    volumes:
  #      - ./Data_Processing/code:/opt/spark-app
  #      - ./Data_Processing/models:/opt/spark
  #    deploy:
  #      resources:
  #        limits:
  #          cpus: '3.0'
  #          memory: 4g
  #    networks:
  #      - tap_project_2
  #        #ipv4_address: 10.0.100.2
  #    command: spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.2,org.elasticsearch:elasticsearch-spark-30_2.12:8.2.0 --master local[*] /opt/spark-app/topic_modelling.py
  #    depends_on:
  #      - data_streamer_kafkaServer
  #    restart: unless-stopped
  #    build:
  #      context: ./Data_Processing
  #      dockerfile: Dockerfile

  # stream ~ hostuxsocial
  #spark_7:
  #    image: tap_project:spark
  #    container_name: spark_7
  #    environment:
  #      - SPARK_ACTION=bash
  #      - SPARK_MASTER_URL=spark://spark-master:7077
  #      - SPARK_DEPLOY_MODE=client
  #      - SPARK_APPLICATION_PYTHON_FILES=/opt/spark-app/topic_modelling.py
  #      - KAFKA_TOPIC=hostuxsocial
  #    # Remember the topic ^
  #      - ELASTIC_HOST_URL=http://elasticsearch:9200
  #    # Configure the Data Indexer dynamically
  #      - KAFKA_SERVER_URL=kafkaServer:9092
  #    # Configure the kafka server dynamically
  #      - ELASTIC_SERVER_HOSTNAME=elasticsearch
  #    # Configure the elasticsearch hostname
  #    volumes:
  #      - ./Data_Processing/code:/opt/spark-app
  #      - ./Data_Processing/models:/opt/spark
  #    deploy:
  #      resources:
  #        limits:
  #          cpus: '3.0'
  #          memory: 4g
  #    networks:
  #      - tap_project_2
  #        #ipv4_address: 10.0.100.2
  #    command: spark-submit --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.2,org.elasticsearch:elasticsearch-spark-30_2.12:8.2.0 --master local[*] /opt/spark-app/topic_modelling.py
  #    depends_on:
  #      - data_streamer_kafkaServer
  #    restart: unless-stopped
  #    build:
  #      context: ./Data_Processing
  #      dockerfile: Dockerfile


  elasticsearch:
    container_name: elasticsearch
    hostname: elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0
    ports:
      - "9200:9200"
      - "9300:9300"      
    environment:
      - node.name=elasticsearch
      - xpack.security.enabled=false
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - cluster.routing.allocation.disk.threshold_enabled=false
    ulimits:
      memlock:
        soft: -1
        hard: -1
    restart: unless-stopped
    depends_on:
      - spark_1
    networks:
      - tap_project_2
  
  kibana:
    image: docker.elastic.co/kibana/kibana:8.2.0
    container_name: kibana
    hostname: kibana
    ports:
      - "5601:5601"
    environment:
      - xpack.security.enabled=false
      - KBN_PATH_CONF=/usr/share/kibana/config
    networks:
      - tap_project_2
    volumes:
      - ./Data_Visualization/kibana.yml:/usr/share/kibana/config/kibana.yml
      - ./Data_Visualization/saved-objects:/usr/share/kibana/saved-objects
    depends_on: 
      - elasticsearch
    restart: unless-stopped
    build:
      context: ./Data_Visualization
      dockerfile: Dockerfile


networks:
  tap_project_2:
    external: true  
    # driver: bridge

volumes:
  datastorage:
    external: true