Comment by synae[he/him]@lemmy.dbzer0.com on Lemmy server setup on lemmy.cafe

docker compose

I’m using a v2 - notice the lack of a dash between docker and compose.

I’ve recently learnt of the default filenames docker compose is trying to source upon invocation and decided to give it a try. The files are:

compose.yml
compose.override.yml

I have split the default docker-compose.yml that lemmy comes with into 2 parts - compose.yml holds pict-rs, postfix and, in my case, gatus. compose.override.yml is responsible for lemmy services only. This is what the files contain:

compose.yml

x-logging: &default-logging
  driver: "json-file"
  options:
    max-size: "20m"
    max-file: "4"

services:
  pictrs:
    image: asonix/pictrs:0.5.0
    user: 991:991
    ports:
      - "127.0.0.1:28394:8080"
    volumes:
      - ./volumes/pictrs:/mnt
    restart: always
    logging: *default-logging
    entrypoint: /sbin/tini -- /usr/local/bin/pict-rs run
    environment:
      - PICTRS__OLD_REPO__PATH=/mnt/sled-repo
      - PICTRS__REPO__TYPE=postgres
      - PICTRS__REPO__URL=postgres://pictrs:<redacted>@psql:5432/pictrs
      - RUST_LOG=warn
      - PICTRS__MEDIA__MAX_FILE_SIZE=1
      - PICTRS__MEDIA__IMAGE__FORMAT=webp
    deploy:
      resources:
        limits:
          memory: 512m
  postfix:
    image: mwader/postfix-relay
    environment:
      - POSTFIX_myhostname=lemmy.cafe
    volumes:
      - ./volumes/postfix:/etc/postfix
    restart: "always"
    logging: *default-logging

  gatus:
    image: twinproduction/gatus
    ports:
      - "8080:8080"
    volumes:
      - ./volumes/gatus:/config
    restart: always
    logging: *default-logging
    deploy:
      resources:
        limits:
          memory: 128M

compose.override.yml is actually a hardlink to the currently active deployment. I have two separate files - compose-green.yml and compose-blue.yml. This allows me to prepare and deploy an upgrade to lemmy while the old version is still running.

compose-green.yml

services:
  lemmy-green:
    image: dessalines/lemmy:0.19.2
    hostname: lemmy-green
    ports:
      - "127.0.1.1:14422:8536"
    restart: always
    logging: *default-logging
    environment:
      - RUST_LOG="warn"
    volumes:
      - ./lemmy.hjson:/config/config.hjson
    # depends_on:
    #   - pictrs
    deploy:
      resources:
        limits:
          # cpus: "0.1"
          memory: 128m
    entrypoint: lemmy_server --disable-activity-sending --disable-scheduled-tasks

  lemmy-federation-green:
    image: dessalines/lemmy:0.19.2
    hostname: lemmy-federation-green
    ports:
      - "127.0.1.1:14423:8536"
    restart: always
    logging: *default-logging
    environment:
      - RUST_LOG="warn,activitypub_federation=info"
    volumes:
      - ./lemmy-federation.hjson:/config/config.hjson
    # depends_on:
    #   - pictrs
    deploy:
      resources:
        limits:
          cpus: "0.2"
          memory: 512m
    entrypoint: lemmy_server --disable-http-server --disable-scheduled-tasks

  lemmy-tasks-green:
    image: dessalines/lemmy:0.19.2
    hostname: lemmy-tasks
    ports:
      - "127.0.1.1:14424:8536"
    restart: always
    logging: *default-logging
    environment:
      - RUST_LOG="info"
    volumes:
      - ./lemmy-tasks.hjson:/config/config.hjson
    # depends_on:
    #   - pictrs
    deploy:
      resources:
        limits:
          cpus: "0.1"
          memory: 128m
    entrypoint: lemmy_server --disable-http-server --disable-activity-sending

#############################################################################

  lemmy-ui-green:
    image: dessalines/lemmy-ui:0.19.2
    ports:
      - "127.0.1.1:17862:1234"
    restart: always
    logging: *default-logging
    environment:
      - LEMMY_UI_LEMMY_INTERNAL_HOST=lemmy-green:8536
      - LEMMY_UI_LEMMY_EXTERNAL_HOST=lemmy.cafe
      - LEMMY_UI_HTTPS=true
    volumes:
      - ./volumes/lemmy-ui/extra_themes:/app/extra_themes
    depends_on:
      - lemmy-green
    deploy:
      resources:
        limits:
          memory: 256m

compose-blue.yml

services:
  lemmy-blue:
    image: dessalines/lemmy:0.19.2-rc.5
    hostname: lemmy-blue
    ports:
      - "127.0.2.1:14422:8536"
    restart: always
    logging: *default-logging
    environment:
      - RUST_LOG="warn"
    volumes:
      - ./lemmy.hjson:/config/config.hjson
    # depends_on:
    #   - pictrs
    deploy:
      resources:
        limits:
          # cpus: "0.1"
          memory: 128m
    entrypoint: lemmy_server --disable-activity-sending --disable-scheduled-tasks

  lemmy-federation-blue:
    image: dessalines/lemmy:0.19.2-rc.5
    hostname: lemmy-federation-blue
    ports:
      - "127.0.2.1:14423:8536"
    restart: always
    logging: *default-logging
    environment:
      - RUST_LOG="warn,activitypub_federation=info"
    volumes:
      - ./lemmy-federation.hjson:/config/config.hjson
    # depends_on:
    #   - pictrs
    deploy:
      resources:
        limits:
          cpus: "0.2"
          memory: 512m
    entrypoint: lemmy_server --disable-http-server --disable-scheduled-tasks

  lemmy-tasks-blue:
    image: dessalines/lemmy:0.19.2-rc.5
    hostname: lemmy-tasks-blue
    ports:
      - "127.0.2.1:14424:8536"
    restart: always
    logging: *default-logging
    environment:
      - RUST_LOG="info"
    volumes:
      - ./lemmy-tasks.hjson:/config/config.hjson
    # depends_on:
    #   - pictrs
    deploy:
      resources:
        limits:
          cpus: "0.1"
          memory: 128m
    entrypoint: lemmy_server --disable-http-server --disable-activity-sending

#############################################################################

  lemmy-ui-blue:
    image: dessalines/lemmy-ui:0.19.2-rc.5
    ports:
      - "127.0.2.1:17862:1234"
    restart: always
    logging: *default-logging
    environment:
      - LEMMY_UI_LEMMY_INTERNAL_HOST=lemmy-blue:8536
      - LEMMY_UI_LEMMY_EXTERNAL_HOST=lemmy.cafe
      - LEMMY_UI_HTTPS=true
    volumes:
      - ./volumes/lemmy-ui/extra_themes:/app/extra_themes
    depends_on:
      - lemmy-blue
    deploy:
      resources:
        limits:
          memory: 256m

The only constant different between the two is the IP address I use to expose them to the host. I’ve tried using ports, but found that it’s much easier to follow it in my mind by sticking to the ports and changing the bound IP.

I also have two nginx configs to reflect the different IP for green/blue deployments, but pasting the whole config here would be a tad too much.

No-downtime upgrade

Let’s say green is the currently active deployment. In that case - edit the compose-blue.yml file to change the version of lemmy on all 4 components - lemmy, federation, tasks and ui. Then bring down the tasks container from the active deployment, activate the whole of blue deployment and link it to be the compose.override.yml. Once the tasks container is done with whatever tasks it’s supposed to do - switch over the nginx config. Et voilà - no downtime upgrade is live!

Now all that’s left to do is tear down the green containers.

docker compose down lemmy-tasks-green
docker compose -f compose-blue.yml up -d
ln -f compose-blue.yml compose.override.yml
# Wait for tasks to finish
ln -sf /etc/nginx/sites-available/lemmy.cafe-blue.conf /etc/sites-enabled/lemmy.cafe.conf
nginx -t && nginx -s reload
docker compose -f compose-green.yml down lemmy-green lemmy-federation-green lemmy-tasks-green lemmy-ui-green

lemmy.hjson

I have also multiplied lemmy.hjson to provide a bit more control.

lemmy.hjson

{
  database: {
    host: "psql"
    port: 5432
    user: "lemmy"
    password: "<redacted>"
    pool_size: 3
  }
  hostname: "lemmy.cafe"
  pictrs: {
    url: "http://pictrs:8080/"
    api_key: "<redacted>"
  }
  email: {
    smtp_server: "postfix:25"
    smtp_from_address: "lemmy@lemmy.cafe"
    tls_type: "none"
  }
}

lemmy-federation.hjson

{
  database: {
    host: "psql"
    port: 5432
    user: "lemmy_federation"
    password: "<redacted>"
    pool_size: 10
  }
  hostname: "lemmy.cafe"
  pictrs: {
    url: "http://pictrs:8080/"
    api_key: "<redacted>"
  }
  email: {
    smtp_server: "postfix:25"
    smtp_from_address: "lemmy@lemmy.cafe"
    tls_type: "none"
  }
  worker_count: 10
  retry_count: 2
}

lemmy-tasks.hjson

{
  database: {
    host: "10.20.0.2"
    port: 5432
    user: "lemmy_tasks"
    password: "<redacted>"
    pool_size: 3
  }
  hostname: "lemmy.cafe"
  pictrs: {
    url: "http://pictrs:8080/"
    api_key: "<redacted>"
  }
  email: {
    smtp_server: "postfix:25"
    smtp_from_address: "lemmy@lemmy.cafe"
    tls_type: "none"
  }
}

I suspect it might be possible to remove pict-rs and/or email config from some of them, but honestly it’s not a big deal and I haven’t had enough time, yet, to look at it.

Future steps

I’d like to script the actual switch-over - it’s really trivial, especially since most of the parts are there already. All I’d really like is apply strict failure mode on the script and see how it behaves; do a few actual upgrades.

Once that happens - I’ll post it here.

So long and thanks for all the fish!

Lemmy server setup on lemmy.cafe

docker compose

No-downtime upgrade

lemmy.hjson

Future steps

graybeard

!graybeard@lemmy.cafe

Community stats

Community moderators