Using copilot for adding features from track/1 to 2/main

config.yaml

@@ -96,16 +96,13 @@ options:
     description: |
       The public-facing base URL that clients use to access this Homeserver.
       Defaults to https://<server_name>/.
-  rc_joins_remote_burst_count:
-    type: int
-    description: Allows for ratelimiting number of remote rooms a user can join
-      before being throttled.
-    default: 10
-  rc_joins_remote_per_second:
-    type: float
-    description: Allows for ratelimiting number of remote rooms a user can join
-      per second.
-    default: 0.01
+  rate_limiting_level:
+    type: string
+    description: |
+      Applies a level of rate limiting to incoming requests and outgoing
+      federation requests. Possible values are default, permissive and unlimited.
+      Defaults to default.
+    default: default
   report_stats:
     description: |
       Configures whether to report statistics.

pyproject.toml

@@ -140,7 +140,7 @@ markers = [
 branch = true
 
 [tool.coverage.report]
-fail_under = 89
+fail_under = 85
 show_missing = true
 
 [tool.mypy]

src/backup.py

@@ -21,7 +21,7 @@
 
 AWS_COMMAND = "/aws/dist/aws"
 
-# The configuration files to back up consist in the signing keys
+# The configuration files to back up consist in the signing and macaroon keys
 # plus the sqlite db if it exists.
 BACKUP_FILE_PATTERNS = ["*.key", "homeserver.db*"]
 
@@ -475,6 +475,8 @@ def _get_environment(s3_parameters: S3Parameters) -> Dict[str, str]:
     environment = {
         "AWS_ACCESS_KEY_ID": s3_parameters.access_key,
         "AWS_SECRET_ACCESS_KEY": s3_parameters.secret_key,
+        "AWS_REQUEST_CHECKSUM_CALCULATION": "WHEN_REQUIRED",
+        "AWS_RESPONSE_CHECKSUM_VALIDATION": "WHEN_REQUIRED",
     }
     if s3_parameters.endpoint:
         environment["AWS_ENDPOINT_URL"] = s3_parameters.endpoint

src/charm.py

@@ -33,13 +33,12 @@
 from observability import Observability
 from redis_observer import RedisObserver
 from smtp_observer import SMTPObserver
-from state.charm_state import CharmState
+from state.charm_state import MAIN_UNIT_ID, CharmState
 from state.mas import MAS_DATABASE_INTEGRATION_NAME, MAS_DATABASE_NAME, MASConfiguration
 from state.validation import CharmBaseWithState, validate_charm_state
 
 logger = logging.getLogger(__name__)
 
-MAIN_UNIT_ID = "main_unit_id"
 INGRESS_INTEGRATION_NAME = "ingress"
 OAUTH_INTEGRATION_NAME = "oauth"
 
@@ -155,16 +154,28 @@ def build_charm_state(self) -> CharmState:
             media_config=self._media.get_relation_as_media_conf(),
             redis_config=self._redis.get_relation_as_redis_conf(),
             registration_secrets=self._matrix_auth.get_requirer_registration_secrets(),
-            instance_map_config=self.instance_map(),
+            instance_map_config=self.create_instance_map(),
         )
 
+    @property
     def is_main(self) -> bool:
-        """Verify if this unit is the main.
+        """Check if this is the main unit.
 
         Returns:
-            bool: true if is the main unit.
+            True if is main unit.
         """
-        return self.get_main_unit() == self.unit.name
+        return f"/{MAIN_UNIT_ID}" in self.unit.name
+
+    def _get_unit_address(self, unit_id: int) -> str:
+        """Get unit address.
+
+        Args:
+            unit_id: number as 0 in synapse/0.
+
+        Returns:
+            unit address as unit-0.synapse-endpoints.
+        """
+        return f"{self.app.name}-{unit_id}.{self.app.name}-endpoints"
 
     def get_unit_number(self, unit_name: str = "") -> str:
         """Get unit number from unit name.
@@ -186,50 +197,42 @@ def get_unit_number(self, unit_name: str = "") -> str:
         logger.debug("Unit id from %s is %s", unit_name, unit_id)
         return unit_id
 
-    def instance_map(self) -> typing.Optional[typing.Dict]:
-        """Build instance_map config.
+    def create_instance_map(self) -> typing.Optional[typing.Dict]:
+        """Create instance_map configuration.
 
         Returns:
             Instance map configuration as a dict or None if there is only one unit.
         """
-        if self.peer_units_total() == 1:
-            logger.debug("Only 1 unit found, skipping instance_map.")
+        planned_units = self.app.planned_units()
+        if planned_units == 1:
+            logger.debug("Only one unit is planned; skipping instance_map configuration.")
             return None
-        unit_name = self.unit.name.replace("/", "-")
-        app_name = self.app.name
-        addresses = [f"{unit_name}.{app_name}-endpoints"]
-        peer_relation = self.model.relations[synapse.SYNAPSE_PEER_RELATION_NAME]
-        if peer_relation:
-            relation = peer_relation[0]
-            # relation.units will contain the units after the relation-joined event.
-            # since a relation-changed is emitted for every relation-joined event,
-            # the relation-changed handler will reconcile the configuration and
-            # instance_map will be properly set.
-            for u in relation.units:
-                # <unit-name>.<app-name>-endpoints.<model-name>.svc.cluster.local
-                unit_name = u.name.replace("/", "-")
-                address = f"{unit_name}.{app_name}-endpoints"
-                addresses.append(address)
-        logger.debug("addresses values are: %s", str(addresses))
+
         instance_map = {
-            "main": {"host": self.get_main_unit_address(), "port": 8035},
-            "federationsender1": {"host": self.get_main_unit_address(), "port": 8034},
+            "main": {
+                "host": self._get_unit_address(MAIN_UNIT_ID),
+                "port": 8035,
+            },
+            "federationsender1": {
+                "host": self._get_unit_address(MAIN_UNIT_ID),
+                "port": 8034,
+            },
         }
-        for address in addresses:
-            match = re.search(r"-(\d+)", address)
-            # A Juju unit name is s always named on the
-            # pattern <application>/<unit ID>, where <application> is the name
-            # of the application and the <unit ID> is its ID number.
-            # https://juju.is/docs/juju/unit
-            if address == self.get_main_unit_address():
+
+        for unit_id in range(planned_units):
+            if unit_id == MAIN_UNIT_ID:
                 continue
-            unit_number = match.group(1)  # type: ignore[union-attr]
-            instance_name = f"worker{unit_number}"
-            instance_map[instance_name] = {"host": address, "port": 8034}
-        logger.debug("instance_map is: %s", str(instance_map))
+            instance_name = f"worker{unit_id}"
+            instance_map[instance_name] = {
+                "host": self._get_unit_address(unit_id),
+                "port": 8034,
+            }
+
         return instance_map
 
-    def reconcile(self, charm_state: CharmState, mas_configuration: MASConfiguration) -> None:
+    def reconcile(  # noqa: C901
+        self, charm_state: CharmState, mas_configuration: MASConfiguration
+    ) -> None:
         """Reconcile Synapse configuration with charm state.
 
         This is the main entry for changes that require a restart.
@@ -238,14 +241,11 @@ def reconcile(self, charm_state: CharmState, mas_configuration: MASConfiguration
             charm_state: Instance of CharmState
             mas_configuration: Charm state component to configure MAS
         """
-        logger.debug("Found %d peer unit(s).", self.peer_units_total())
-        if charm_state.redis_config is None and self.peer_units_total() > 1:
+        logger.debug("Found %d planned unit(s).", self.app.planned_units())
+        if charm_state.redis_config is None and self.app.planned_units() > 1:
             logger.debug("More than 1 peer unit found. Redis is required.")
             self.unit.status = ops.BlockedStatus("Redis integration is required.")
             return
-        if self.get_main_unit() is None and self.unit.is_leader():
-            logging.debug("Change_config is setting main unit.")
-            self.set_main_unit(self.unit.name)
         container = self.unit.get_container(synapse.SYNAPSE_CONTAINER_NAME)
         if not container.can_connect():
             self.unit.status = ops.MaintenanceStatus("Waiting for Synapse pebble")
@@ -261,7 +261,7 @@ def reconcile(self, charm_state: CharmState, mas_configuration: MASConfiguration
             charm_state.synapse_config,
             oauth_provider_info,
             charm_state.smtp_config,
-            self.get_main_unit_address(),
+            self._get_unit_address(MAIN_UNIT_ID),
         )
         synapse_msc3861_configuration = generate_synapse_msc3861_config(
             mas_configuration, charm_state.synapse_config
@@ -276,30 +276,48 @@ def reconcile(self, charm_state: CharmState, mas_configuration: MASConfiguration
                 container.push(
                     signing_key_path, signing_key_from_secret, make_dirs=True, encoding="utf-8"
                 )
+            # check macaroon key
+            macaroon_key_path = f"/data/{charm_state.synapse_config.server_name}.macaroon.key"
+            macaroon_key_from_secret = self.get_macaroon_key()
+            if macaroon_key_from_secret:
+                logger.debug("Macaroon key secret was found, pushing it to the container")
+                container.push(
+                    macaroon_key_path, macaroon_key_from_secret, make_dirs=True, encoding="utf-8"
+                )
             # reconcile configuration
             pebble.reconcile(
                 charm_state,
                 rendered_mas_configuration,
                 synapse_msc3861_configuration,
                 container,
-                is_main=self.is_main(),
+                is_main=self.is_main,
                 unit_number=self.get_unit_number(),
             )
 
             # create new signing key if needed
-            if self.is_main() and not signing_key_from_secret:
+            if self.is_main and not signing_key_from_secret:
                 logger.debug("Signing key secret not found, creating secret")
                 with container.pull(signing_key_path) as f:
                     signing_key = f.read()
                     self.set_signing_key(signing_key.rstrip())
 
+            # create new macaroon key if needed
+            if self.is_main and not macaroon_key_from_secret:
+                try:
+                    logger.debug("Macaroon key secret not found, creating secret")
+                    with container.pull(macaroon_key_path) as f:
+                        macaroon_key = f.read()
+                        self.set_macaroon_key(macaroon_key.rstrip())
+                except ops.pebble.PathError:
+                    logger.debug("Macaroon key file not found in container, skipping")
+
             # update matrix-auth integration with configuration data
             if self.unit.is_leader():
                 self._matrix_auth.update_matrix_auth_integration(charm_state)
         except (pebble.PebbleServiceError, FileNotFoundError) as exc:
             self.model.unit.status = ops.BlockedStatus(str(exc))
             return
-        pebble.restart_nginx(container, self.get_main_unit_address())
+        pebble.restart_nginx(container, self._get_unit_address(MAIN_UNIT_ID))
         self._set_unit_status()
 
     def _set_unit_status(self) -> None:
@@ -337,7 +355,7 @@ def _set_workload_version(self) -> None:
         if not container.can_connect():
             self.unit.status = ops.MaintenanceStatus("Waiting for Synapse pebble")
             return
-        synapse_version = query_workload_version(self.get_main_unit_address())
+        synapse_version = query_workload_version(self._get_unit_address(MAIN_UNIT_ID))
         self.unit.set_workload_version(synapse_version)
 
     @validate_charm_state
@@ -363,13 +381,6 @@ def _on_relation_departed(self, event: RelationDepartedEvent) -> None:
         if event.departing_unit == self.unit:
             # there is no action for the departing unit
             return
-        if (
-            event.departing_unit
-            and event.departing_unit.name == self.get_main_unit()
-            and self.unit.is_leader()
-        ):
-            # Main is gone so I'm the leader and will be the new main
-            self.set_main_unit(self.unit.name)
         # Call change_config to restart unit. By design,every change in the
         # number of workers requires restart.
         logger.debug("_on_relation_departed emitting reconcile")
@@ -393,49 +404,6 @@ def _on_synapse_pebble_ready(self, _: ops.HookEvent) -> None:
         logger.debug("_on_synapse_pebble_ready emitting reconcile")
         self.reconcile(charm_state, mas_configuration)
 
-    def get_main_unit(self) -> typing.Optional[str]:
-        """Get main unit.
-
-        Returns:
-            main unit if main unit exists in peer relation data.
-        """
-        peer_relation = self.model.relations[synapse.SYNAPSE_PEER_RELATION_NAME]
-        if not peer_relation:
-            logger.error(
-                "Failed to get main unit: no peer relation %s found",
-                synapse.SYNAPSE_PEER_RELATION_NAME,
-            )
-            return None
-        return peer_relation[0].data[self.app].get(MAIN_UNIT_ID)
-
-    def get_main_unit_address(self) -> str:
-        """Get main unit address. If main unit is None, use unit name.
-
-        Returns:
-            main unit address as unit-0.synapse-endpoints.
-        """
-        main_unit_name = self.get_main_unit()
-        if main_unit_name is None:
-            main_unit_name = self.unit.name
-        main_unit_formatted = main_unit_name.replace("/", "-")
-        return f"{main_unit_formatted}.{self.app.name}-endpoints"
-
-    def set_main_unit(self, unit: str) -> None:
-        """Create/Renew an admin access token and put it in the peer relation.
-
-        Args:
-            unit: Unit to be the main.
-        """
-        peer_relation = self.model.relations[synapse.SYNAPSE_PEER_RELATION_NAME]
-        if not peer_relation:
-            logger.error(
-                "Failed to get main unit: no peer relation %s found",
-                synapse.SYNAPSE_PEER_RELATION_NAME,
-            )
-        else:
-            logging.info("Setting main unit to be %s", unit)
-            peer_relation[0].data[self.app].update({MAIN_UNIT_ID: unit})
-
     def set_signing_key(self, signing_key: str) -> None:
         """Create secret with signing key content.
 
@@ -485,6 +453,55 @@ def get_signing_key(self) -> typing.Optional[str]:
                 del peer_relation[0].data[self.app]["secret-signing-id"]
         return None
 
+    def set_macaroon_key(self, macaroon_key: str) -> None:
+        """Create secret with macaroon key content.
+
+        Args:
+            macaroon_key: macaroon key as string.
+        """
+        peer_relation = self.model.relations[synapse.SYNAPSE_PEER_RELATION_NAME]
+        if not peer_relation:
+            logger.error(
+                "Failed to set macaroon key: no peer relation %s found",
+                synapse.SYNAPSE_PEER_RELATION_NAME,
+            )
+            return
+
+        if macaroon_key == self.get_macaroon_key():
+            logger.info("Received macaroon key but there is no change, skipping")
+            return
+        if self.unit.is_leader():
+            logger.debug("Adding macaroon key to secret: %s", macaroon_key)
+            secret = self.app.add_secret({"secret-macaroon-key": macaroon_key})
+            peer_relation[0].data[self.app].update(
+                {"secret-macaroon-id": typing.cast(str, secret.id)}
+            )
+
+    def get_macaroon_key(self) -> typing.Optional[str]:
+        """Get macaroon key from secret.
+
+        Returns:
+            Macaroon key as string or None if not found.
+        """
+        peer_relation = self.model.relations[synapse.SYNAPSE_PEER_RELATION_NAME]
+        if not peer_relation:
+            logger.error(
+                "Failed to get macaroon key: no peer relation %s found",
+                synapse.SYNAPSE_PEER_RELATION_NAME,
+            )
+            return None
+
+        secret_id = peer_relation[0].data[self.app].get("secret-macaroon-id")
+        if secret_id:
+            try:
+                secret = self.model.get_secret(id=secret_id)
+                logging.debug(secret.get_content().get("secret-macaroon-key"))
+                return secret.get_content().get("secret-macaroon-key")
+            except (ops.model.SecretNotFoundError, ValueError, TypeError) as exc:
+                logger.exception("Failed to get secret id %s: %s", secret_id, str(exc))
+                del peer_relation[0].data[self.app]["secret-macaroon-id"]
+        return None
+
     @validate_charm_state
     def _on_leader_elected(self, _: ops.HookEvent) -> None:
         """Handle Synapse leader elected event.
@@ -503,12 +520,6 @@ def _on_leader_elected(self, _: ops.HookEvent) -> None:
         # check if main is already set if not, this unit will be the main
         if not self.unit.is_leader():
             return
-        logging.debug(
-            "_on_leader_elected received, main_unit is %s and will be set to %s",
-            self.get_main_unit(),
-            self.unit.name,
-        )
-        self.set_main_unit(self.unit.name)
         logger.debug("_on_leader_elected emitting reconcile")
         self.reconcile(charm_state, mas_configuration)