Cleanup Python 3.8 leftovers (#17967)

Some small cleanups after Python3.8 became EOL.

- Move some type imports from `typing_extensions` to `typing`
- Remove the `abi3-py38` feature from pyo3

### Pull Request Checklist

<!-- Please read
https://element-hq.github.io/synapse/latest/development/contributing_guide.html
before submitting your pull request -->

* [x] Pull request is based on the develop branch
* [x] Pull request includes a [changelog
file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog).
The entry should:
- Be a short description of your change which makes sense to users.
"Fixed a bug that prevented receiving messages from other servers."
instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
  - Use markdown where necessary, mostly for `code blocks`.
  - End with either a period (.) or an exclamation mark (!).
  - Start with a capital letter.
- Feel free to credit yourself, by adding a sentence "Contributed by
@github_username." or "Contributed by [Your Name]." to the end of the
entry.
* [x] [Code
style](https://element-hq.github.io/synapse/latest/code_style.html) is
correct
(run the
[linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))

---------

Co-authored-by: Quentin Gliech <quenting@element.io>

changelog.d/17967.misc

@@ -0,0 +1 @@
+Python 3.8 EOL: compile native extensions with the 3.9 ABI and use typing hints from the standard library.

rust/Cargo.toml

@@ -34,7 +34,7 @@ pyo3 = { version = "0.23.2", features = [
     "macros",
     "anyhow",
     "abi3",
-    "abi3-py38",
+    "abi3-py39",
 ] }
 pyo3-log = "0.12.0"
 pythonize = "0.23.0"

synapse/_scripts/synapse_port_db.py

@@ -42,12 +42,12 @@ from typing import (
     Set,
     Tuple,
     Type,
+    TypedDict,
     TypeVar,
     cast,
 )
 
 import yaml
-from typing_extensions import TypedDict
 
 from twisted.internet import defer, reactor as reactor_
 

synapse/api/auth/__init__.py

@@ -18,9 +18,7 @@
 # [This file includes modifications made by New Vector Limited]
 #
 #
-from typing import TYPE_CHECKING, Optional, Tuple
-
-from typing_extensions import Protocol
+from typing import TYPE_CHECKING, Optional, Protocol, Tuple
 
 from twisted.web.server import Request
 

synapse/event_auth.py

@@ -32,6 +32,7 @@ from typing import (
     Mapping,
     MutableMapping,
     Optional,
+    Protocol,
     Set,
     Tuple,
     Union,
@@ -41,7 +42,6 @@ from typing import (
 from canonicaljson import encode_canonical_json
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import SignatureVerifyException, verify_signed_json
-from typing_extensions import Protocol
 from unpaddedbase64 import decode_base64
 
 from synapse.api.constants import (

synapse/events/__init__.py

@@ -30,6 +30,7 @@ from typing import (
     Generic,
     Iterable,
     List,
+    Literal,
     Optional,
     Tuple,
     Type,
@@ -39,7 +40,6 @@ from typing import (
 )
 
 import attr
-from typing_extensions import Literal
 from unpaddedbase64 import encode_base64
 
 from synapse.api.constants import EventTypes, RelationTypes

synapse/federation/sender/__init__.py

@@ -139,13 +139,13 @@ from typing import (
     Hashable,
     Iterable,
     List,
+    Literal,
     Optional,
     Tuple,
 )
 
 import attr
 from prometheus_client import Counter
-from typing_extensions import Literal
 
 from twisted.internet import defer
 

synapse/federation/transport/server/__init__.py

@@ -20,9 +20,7 @@
 #
 #
 import logging
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Type
-
-from typing_extensions import Literal
+from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Tuple, Type
 
 from synapse.api.errors import FederationDeniedError, SynapseError
 from synapse.federation.transport.server._base import (

synapse/federation/transport/server/federation.py

@@ -24,6 +24,7 @@ from typing import (
     TYPE_CHECKING,
     Dict,
     List,
+    Literal,
     Mapping,
     Optional,
     Sequence,
@@ -32,8 +33,6 @@ from typing import (
     Union,
 )
 
-from typing_extensions import Literal
-
 from synapse.api.constants import Direction, EduTypes
 from synapse.api.errors import Codes, SynapseError
 from synapse.api.room_versions import RoomVersions

synapse/handlers/directory.py

@@ -21,9 +21,7 @@
 
 import logging
 import string
-from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence
-
-from typing_extensions import Literal
+from typing import TYPE_CHECKING, Iterable, List, Literal, Optional, Sequence
 
 from synapse.api.constants import MAX_ALIAS_LENGTH, EventTypes
 from synapse.api.errors import (

synapse/handlers/e2e_room_keys.py

@@ -20,9 +20,7 @@
 #
 
 import logging
-from typing import TYPE_CHECKING, Dict, Optional, cast
-
-from typing_extensions import Literal
+from typing import TYPE_CHECKING, Dict, Literal, Optional, cast
 
 from synapse.api.errors import (
     Codes,

synapse/handlers/oidc.py

@@ -31,6 +31,7 @@ from typing import (
     List,
     Optional,
     Type,
+    TypedDict,
     TypeVar,
     Union,
 )
@@ -52,7 +53,6 @@ from pymacaroons.exceptions import (
     MacaroonInitException,
     MacaroonInvalidSignatureException,
 )
-from typing_extensions import TypedDict
 
 from twisted.web.client import readBody
 from twisted.web.http_headers import Headers

synapse/handlers/register.py

@@ -23,10 +23,9 @@
 """Contains functions for registering clients."""
 
 import logging
-from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple
+from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, TypedDict
 
 from prometheus_client import Counter
-from typing_extensions import TypedDict
 
 from synapse import types
 from synapse.api.constants import (

synapse/handlers/sso.py

@@ -33,12 +33,12 @@ from typing import (
     Mapping,
     NoReturn,
     Optional,
+    Protocol,
     Set,
 )
 from urllib.parse import urlencode
 
 import attr
-from typing_extensions import Protocol
 
 from twisted.web.iweb import IRequest
 from twisted.web.server import Request

synapse/http/client.py

@@ -31,6 +31,7 @@ from typing import (
     List,
     Mapping,
     Optional,
+    Protocol,
     Tuple,
     Union,
 )
@@ -40,7 +41,6 @@ import treq
 from canonicaljson import encode_canonical_json
 from netaddr import AddrFormatError, IPAddress, IPSet
 from prometheus_client import Counter
-from typing_extensions import Protocol
 from zope.interface import implementer
 
 from OpenSSL import SSL

synapse/http/matrixfederationclient.py

@@ -34,6 +34,7 @@ from typing import (
     Dict,
     Generic,
     List,
+    Literal,
     Optional,
     TextIO,
     Tuple,
@@ -48,7 +49,6 @@ import treq
 from canonicaljson import encode_canonical_json
 from prometheus_client import Counter
 from signedjson.sign import sign_json
-from typing_extensions import Literal
 
 from twisted.internet import defer
 from twisted.internet.error import DNSLookupError

synapse/http/server.py

@@ -39,6 +39,7 @@ from typing import (
     List,
     Optional,
     Pattern,
+    Protocol,
     Tuple,
     Union,
 )
@@ -46,7 +47,6 @@ from typing import (
 import attr
 import jinja2
 from canonicaljson import encode_canonical_json
-from typing_extensions import Protocol
 from zope.interface import implementer
 
 from twisted.internet import defer, interfaces

synapse/http/servlet.py

@@ -28,6 +28,7 @@ from http import HTTPStatus
 from typing import (
     TYPE_CHECKING,
     List,
+    Literal,
     Mapping,
     Optional,
     Sequence,
@@ -37,8 +38,6 @@ from typing import (
     overload,
 )
 
-from typing_extensions import Literal
-
 from twisted.web.server import Request
 
 from synapse._pydantic_compat import (

synapse/logging/context.py

@@ -40,6 +40,7 @@ from typing import (
     Any,
     Awaitable,
     Callable,
+    Literal,
     Optional,
     Tuple,
     Type,
@@ -49,7 +50,7 @@ from typing import (
 )
 
 import attr
-from typing_extensions import Literal, ParamSpec
+from typing_extensions import ParamSpec
 
 from twisted.internet import defer, threads
 from twisted.python.threadpool import ThreadPool

synapse/logging/filter.py

@@ -19,8 +19,7 @@
 #
 #
 import logging
-
-from typing_extensions import Literal
+from typing import Literal
 
 
 class MetadataFilter(logging.Filter):

synapse/metrics/jemalloc.py

@@ -23,11 +23,10 @@ import ctypes
 import logging
 import os
 import re
-from typing import Iterable, Optional, overload
+from typing import Iterable, Literal, Optional, overload
 
 import attr
 from prometheus_client import REGISTRY, Metric
-from typing_extensions import Literal
 
 from synapse.metrics import GaugeMetricFamily
 from synapse.metrics._types import Collector

synapse/module_api/callbacks/spamchecker_callbacks.py

@@ -29,15 +29,13 @@ from typing import (
     Callable,
     Collection,
     List,
+    Literal,
     Optional,
     Tuple,
     Union,
     cast,
 )
 
-# `Literal` appears with Python 3.8.
-from typing_extensions import Literal
-
 import synapse
 from synapse.api.errors import Codes
 from synapse.logging.opentracing import trace

synapse/push/push_types.py

@@ -18,9 +18,7 @@
 # [This file includes modifications made by New Vector Limited]
 #
 #
-from typing import List, Optional
-
-from typing_extensions import TypedDict
+from typing import List, Optional, TypedDict
 
 
 class EmailReason(TypedDict, total=False):

synapse/rest/client/account.py

@@ -21,11 +21,10 @@
 #
 import logging
 import random
-from typing import TYPE_CHECKING, List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Literal, Optional, Tuple
 from urllib.parse import urlparse
 
 import attr
-from typing_extensions import Literal
 
 from twisted.web.server import Request
 

synapse/rest/client/directory.py

@@ -20,9 +20,7 @@
 #
 
 import logging
-from typing import TYPE_CHECKING, List, Optional, Tuple
-
-from typing_extensions import Literal
+from typing import TYPE_CHECKING, List, Literal, Optional, Tuple
 
 from twisted.web.server import Request
 

synapse/rest/client/login.py

@@ -30,11 +30,10 @@ from typing import (
     List,
     Optional,
     Tuple,
+    TypedDict,
     Union,
 )
 
-from typing_extensions import TypedDict
-
 from synapse.api.constants import ApprovalNoticeMedium
 from synapse.api.errors import (
     Codes,

synapse/state/v2.py

@@ -29,15 +29,15 @@ from typing import (
     Generator,
     Iterable,
     List,
+    Literal,
     Optional,
+    Protocol,
     Sequence,
     Set,
     Tuple,
     overload,
 )
 
-from typing_extensions import Literal, Protocol
-
 from synapse import event_auth
 from synapse.api.constants import EventTypes
 from synapse.api.errors import AuthError

synapse/storage/database.py

@@ -35,6 +35,7 @@ from typing import (
     Iterable,
     Iterator,
     List,
+    Literal,
     Mapping,
     Optional,
     Sequence,
@@ -47,7 +48,7 @@ from typing import (
 
 import attr
 from prometheus_client import Counter, Histogram
-from typing_extensions import Concatenate, Literal, ParamSpec
+from typing_extensions import Concatenate, ParamSpec
 
 from twisted.enterprise import adbapi
 from twisted.internet.interfaces import IReactorCore

synapse/storage/databases/main/client_ips.py

@@ -20,10 +20,19 @@
 #
 
 import logging
-from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    TypedDict,
+    Union,
+    cast,
+)
 
 import attr
-from typing_extensions import TypedDict
 
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.storage._base import SQLBaseStore

synapse/storage/databases/main/devices.py

@@ -27,6 +27,7 @@ from typing import (
     Dict,
     Iterable,
     List,
+    Literal,
     Mapping,
     Optional,
     Set,
@@ -35,7 +36,6 @@ from typing import (
 )
 
 from canonicaljson import encode_canonical_json
-from typing_extensions import Literal
 
 from synapse.api.constants import EduTypes
 from synapse.api.errors import Codes, StoreError

synapse/storage/databases/main/e2e_room_keys.py

@@ -19,9 +19,18 @@
 #
 #
 
-from typing import TYPE_CHECKING, Dict, Iterable, List, Mapping, Optional, Tuple, cast
-
-from typing_extensions import Literal, TypedDict
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    Tuple,
+    TypedDict,
+    cast,
+)
 
 from synapse.api.errors import StoreError
 from synapse.logging.opentracing import log_kv, trace

synapse/storage/databases/main/end_to_end_keys.py

@@ -27,6 +27,7 @@ from typing import (
     Dict,
     Iterable,
     List,
+    Literal,
     Mapping,
     Optional,
     Sequence,
@@ -39,7 +40,6 @@ from typing import (
 
 import attr
 from canonicaljson import encode_canonical_json
-from typing_extensions import Literal
 
 from synapse.api.constants import DeviceKeyAlgorithms
 from synapse.appservice import (

synapse/storage/databases/main/events.py

@@ -35,12 +35,12 @@ from typing import (
     Sequence,
     Set,
     Tuple,
+    TypedDict,
     cast,
 )
 
 import attr
 from prometheus_client import Counter
-from typing_extensions import TypedDict
 
 import synapse.metrics
 from synapse.api.constants import (

synapse/storage/databases/main/events_worker.py

@@ -30,6 +30,7 @@ from typing import (
     Dict,
     Iterable,
     List,
+    Literal,
     Mapping,
     MutableMapping,
     Optional,
@@ -41,7 +42,6 @@ from typing import (
 
 import attr
 from prometheus_client import Gauge
-from typing_extensions import Literal
 
 from twisted.internet import defer
 

synapse/storage/databases/main/stream.py

@@ -50,6 +50,7 @@ from typing import (
     Dict,
     Iterable,
     List,
+    Literal,
     Mapping,
     Optional,
     Protocol,
@@ -61,7 +62,7 @@ from typing import (
 
 import attr
 from immutabledict import immutabledict
-from typing_extensions import Literal, assert_never
+from typing_extensions import assert_never
 
 from twisted.internet import defer
 

synapse/storage/databases/main/user_directory.py

@@ -31,6 +31,7 @@ from typing import (
     Sequence,
     Set,
     Tuple,
+    TypedDict,
     cast,
 )
 
@@ -44,8 +45,6 @@ try:
 except ModuleNotFoundError:
     USE_ICU = False
 
-from typing_extensions import TypedDict
-
 from synapse.api.errors import StoreError
 from synapse.util.stringutils import non_null_str_or_none
 

synapse/storage/types.py

@@ -26,14 +26,13 @@ from typing import (
     List,
     Mapping,
     Optional,
+    Protocol,
     Sequence,
     Tuple,
     Type,
     Union,
 )
 
-from typing_extensions import Protocol
-
 """
 Some very basic protocol definitions for the DB-API2 classes specified in PEP-249
 """

synapse/types/__init__.py

@@ -40,6 +40,7 @@ from typing import (
     Set,
     Tuple,
     Type,
+    TypedDict,
     TypeVar,
     Union,
     overload,
@@ -49,7 +50,7 @@ import attr
 from immutabledict import immutabledict
 from signedjson.key import decode_verify_key_bytes
 from signedjson.types import VerifyKey
-from typing_extensions import Self, TypedDict
+from typing_extensions import Self
 from unpaddedbase64 import decode_base64
 from zope.interface import Interface
 

synapse/util/async_helpers.py

@@ -41,6 +41,7 @@ from typing import (
     Hashable,
     Iterable,
     List,
+    Literal,
     Optional,
     Set,
     Tuple,
@@ -51,7 +52,7 @@ from typing import (
 )
 
 import attr
-from typing_extensions import Concatenate, Literal, ParamSpec, Unpack
+from typing_extensions import Concatenate, ParamSpec, Unpack
 
 from twisted.internet import defer
 from twisted.internet.defer import CancelledError

synapse/util/caches/dictionary_cache.py

@@ -21,10 +21,19 @@
 import enum
 import logging
 import threading
-from typing import Dict, Generic, Iterable, Optional, Set, Tuple, TypeVar, Union
+from typing import (
+    Dict,
+    Generic,
+    Iterable,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    TypeVar,
+    Union,
+)
 
 import attr
-from typing_extensions import Literal
 
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.caches.treecache import TreeCache

synapse/util/caches/expiringcache.py

@@ -21,10 +21,9 @@
 
 import logging
 from collections import OrderedDict
-from typing import Any, Generic, Iterable, Optional, TypeVar, Union, overload
+from typing import Any, Generic, Iterable, Literal, Optional, TypeVar, Union, overload
 
 import attr
-from typing_extensions import Literal
 
 from twisted.internet import defer
 

synapse/util/caches/lrucache.py

@@ -34,6 +34,7 @@ from typing import (
     Generic,
     Iterable,
     List,
+    Literal,
     Optional,
     Set,
     Tuple,
@@ -44,8 +45,6 @@ from typing import (
     overload,
 )
 
-from typing_extensions import Literal
-
 from twisted.internet import reactor
 from twisted.internet.interfaces import IReactorTime
 

synapse/util/iterutils.py

@@ -30,14 +30,13 @@ from typing import (
     Iterator,
     List,
     Mapping,
+    Protocol,
     Set,
     Sized,
     Tuple,
     TypeVar,
 )
 
-from typing_extensions import Protocol
-
 T = TypeVar("T")
 S = TypeVar("S", bound="_SelfSlice")
 

synapse/util/macaroons.py

@@ -22,12 +22,11 @@
 
 """Utilities for manipulating macaroons"""
 
-from typing import Callable, Optional
+from typing import Callable, Literal, Optional
 
 import attr
 import pymacaroons
 from pymacaroons.exceptions import MacaroonVerificationFailedException
-from typing_extensions import Literal
 
 from synapse.util import Clock, stringutils
 

synapse/util/metrics.py

@@ -22,10 +22,19 @@
 import logging
 from functools import wraps
 from types import TracebackType
-from typing import Awaitable, Callable, Dict, Generator, Optional, Type, TypeVar
+from typing import (
+    Awaitable,
+    Callable,
+    Dict,
+    Generator,
+    Optional,
+    Protocol,
+    Type,
+    TypeVar,
+)
 
 from prometheus_client import CollectorRegistry, Counter, Metric
-from typing_extensions import Concatenate, ParamSpec, Protocol
+from typing_extensions import Concatenate, ParamSpec
 
 from synapse.logging.context import (
     ContextResourceUsage,

tests/media/test_media_storage.py

@@ -23,14 +23,13 @@ import shutil
 import tempfile
 from binascii import unhexlify
 from io import BytesIO
-from typing import Any, BinaryIO, ClassVar, Dict, List, Optional, Tuple, Union
+from typing import Any, BinaryIO, ClassVar, Dict, List, Literal, Optional, Tuple, Union
 from unittest.mock import MagicMock, Mock, patch
 from urllib import parse
 
 import attr
 from parameterized import parameterized, parameterized_class
 from PIL import Image as Image
-from typing_extensions import Literal
 
 from twisted.internet import defer
 from twisted.internet.defer import Deferred

tests/metrics/test_metrics.py

@@ -19,12 +19,11 @@
 #
 #
 from importlib import metadata
-from typing import Dict, Tuple
+from typing import Dict, Protocol, Tuple
 from unittest.mock import patch
 
 from pkg_resources import parse_version
 from prometheus_client.core import Sample
-from typing_extensions import Protocol
 
 from synapse.app._base import _set_prometheus_client_use_created_metrics
 from synapse.metrics import REGISTRY, InFlightGauge, generate_latest

tests/rest/client/test_login.py

@@ -27,6 +27,7 @@ from typing import (
     Collection,
     Dict,
     List,
+    Literal,
     Optional,
     Tuple,
     Union,
@@ -35,7 +36,6 @@ from unittest.mock import Mock
 from urllib.parse import urlencode
 
 import pymacaroons
-from typing_extensions import Literal
 
 from twisted.test.proto_helpers import MemoryReactor
 from twisted.web.resource import Resource

tests/rest/client/test_media.py

@@ -24,14 +24,13 @@ import json
 import os
 import re
 import shutil
-from typing import Any, BinaryIO, Dict, List, Optional, Sequence, Tuple, Type
+from typing import Any, BinaryIO, ClassVar, Dict, List, Optional, Sequence, Tuple, Type
 from unittest.mock import MagicMock, Mock, patch
 from urllib import parse
 from urllib.parse import quote, urlencode
 
 from parameterized import parameterized, parameterized_class
 from PIL import Image as Image
-from typing_extensions import ClassVar
 
 from twisted.internet import defer
 from twisted.internet._resolver import HostResolution

tests/rest/client/test_models.py

@@ -19,8 +19,7 @@
 #
 #
 import unittest as stdlib_unittest
-
-from typing_extensions import Literal
+from typing import Literal
 
 from synapse._pydantic_compat import BaseModel, ValidationError
 from synapse.types.rest.client import EmailRequestTokenBody

tests/rest/client/test_rooms.py

@@ -25,12 +25,11 @@
 
 import json
 from http import HTTPStatus
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Union
 from unittest.mock import AsyncMock, Mock, call, patch
 from urllib import parse as urlparse
 
 from parameterized import param, parameterized
-from typing_extensions import Literal
 
 from twisted.test.proto_helpers import MemoryReactor
 

tests/rest/client/utils.py

@@ -31,6 +31,7 @@ from typing import (
     AnyStr,
     Dict,
     Iterable,
+    Literal,
     Mapping,
     MutableMapping,
     Optional,
@@ -40,7 +41,6 @@ from typing import (
 from urllib.parse import urlencode
 
 import attr
-from typing_extensions import Literal
 
 from twisted.test.proto_helpers import MemoryReactorClock
 from twisted.web.server import Site

tests/unittest.py

@@ -40,6 +40,7 @@ from typing import (
     Mapping,
     NoReturn,
     Optional,
+    Protocol,
     Tuple,
     Type,
     TypeVar,
@@ -50,7 +51,7 @@ from unittest.mock import Mock, patch
 import canonicaljson
 import signedjson.key
 import unpaddedbase64
-from typing_extensions import Concatenate, ParamSpec, Protocol
+from typing_extensions import Concatenate, ParamSpec
 
 from twisted.internet.defer import Deferred, ensureDeferred
 from twisted.python.failure import Failure

tests/util/test_linearizer.py

@@ -19,9 +19,7 @@
 #
 #
 
-from typing import Hashable, Tuple
-
-from typing_extensions import Protocol
+from typing import Hashable, Protocol, Tuple
 
 from twisted.internet import defer, reactor
 from twisted.internet.base import ReactorBase

tests/utils.py

@@ -28,6 +28,7 @@ from typing import (
     Callable,
     Dict,
     List,
+    Literal,
     Optional,
     Tuple,
     Type,
@@ -37,7 +38,7 @@ from typing import (
 )
 
 import attr
-from typing_extensions import Literal, ParamSpec
+from typing_extensions import ParamSpec
 
 from synapse.api.constants import EventTypes
 from synapse.api.room_versions import RoomVersions