debian package 0.2-1

-----BEGIN PGP SIGNATURE-----
 
 iQJYBAABCgBCFiEEcdms641aWv8vJSXMIcx4mUG+20gFAmUhLrEkHHN0ZWZhbi5i
 dWVobGVyQHRpay51bmktc3R1dHRnYXJ0LmRlAAoJECHMeJlBvttImpgQAJ0qSkbQ
 ieZ1Fwc1Vtllz4+6zBZKZuHxN1q+SwNgDudWwuievC2EMDoWAIpbODZFjVrpeGfh
 8dtFO336xC2vzAT/+hjHp2+9a22HnkwuUUpYFl1KV3LqanwQIaLcyGGlOWVvuenG
 mNvP49I7w+CHpeh9D6qVfnrJkIeRi8DwAaNB0WCSZRggaYJHNFyNNHvaBZsuFzKj
 Z8VIcGTyWcib/N4oPYJfkftEiTyrG6Op0PGf7Q3OhpSoqrmBGlQg+ojhTkHBFIM/
 1vpJi6vXVRofxhZ7ogyu9az8X5AUxZdWNeQJhPBORRWj9NxXWBzPlgztH6IsM4FZ
 mlRytAtb5B2iTC4rFUmSAkWAbiurHTB0VLl9CtAyif7KAXT56IoHBDYr+ZEjSSAu
 453GieRNnY2O+6SrpOXVkIb+8L6TlzArwVS6ythxJPiclJRMcBSf/JPAMGX7mzxP
 MHay2lapcmkJLqvykrIwlt3NmzvvO9ApVDrPSZCJs3KKXoRHGOlT5s6jlCSts6rR
 0H/+1oBY8geqUbStTSl2oA9C/dyte6CPfH3NcuR/NAfPXqaG3Hmze8+sFepQg4Bs
 I3UCSi0hbTCllblRj6dhRw+lWeL/7Ur8d8UVien5eWjDUOpCN4Chnl98wukcNVcm
 Arv0a+jjVKObdMz4gmYNLnkH4dAeJ5x3AuGW
 =aDZ5
 -----END PGP SIGNATURE-----

Merge tag 'v0.2' into debian

pqm version 0.2

debian/changelog

@@ -1,3 +1,11 @@
+pqm (0.2-1) unstable; urgency=medium
+
+  * fix json stream decode performance issue
+  * add __slots__ to most classes
+  * require python >= 3.10
+
+ -- Stefan Bühler <stefan.buehler@tik.uni-stuttgart.de>  Sat, 07 Oct 2023 12:11:28 +0200
+
 pqm (0.1-1) unstable; urgency=medium
 
   * Initial release.

debian/control

@@ -11,7 +11,7 @@ Homepage: https://git-nks-public.tik.uni-stuttgart.de/mail/pqm
 Package: pqm
 Architecture: all
 Depends:
- python3:any,
+ python3:any (>= 3.10),
  python3-trio,
  python3-pyparsing,
  python3-yaml,

pqm

@@ -33,7 +33,7 @@ import yaml
 T = typing.TypeVar('T')
 
 
-@dataclasses.dataclass
+@dataclasses.dataclass(slots=True)
 class Config:
     # if not remote_sources are configured -> use local queue
     remote_sources: dict[str, str] = dataclasses.field(default_factory=dict)
@@ -157,7 +157,7 @@ async def trio_parallel_ordered(
         await tpo.close()
 
 
-@dataclasses.dataclass
+@dataclasses.dataclass(slots=True)
 class Recipient:
     """Recipient in a postfix mail"""
     address: str
@@ -184,7 +184,22 @@ class QueueName(enum.Enum):
 ALL_QUEUE_NAMES: set[QueueName] = set(QueueName)
 
 
-@dataclasses.dataclass
+def json_decode_stream(data: str):
+    decoder = json.JSONDecoder()
+    data_len = len(data)
+    data_pos = 0
+    while data_len > data_pos and data[data_len-1].isspace():
+        data_len -= 1
+    while True:
+        while data_pos < data_len and data[data_pos].isspace():
+            data_pos += 1
+        if data_pos >= data_len:
+            return
+        obj, data_pos = decoder.raw_decode(data, data_pos)
+        yield obj
+
+
+@dataclasses.dataclass(slots=True)
 class Mail:
     """Metadata for mail in postfix queue"""
     queue_name: QueueName
@@ -209,11 +224,7 @@ class Mail:
     @staticmethod
     def read_postqueue_json(data: str, id_prefix: str = '') -> list[Mail]:
         queue = []
-        decoder = json.JSONDecoder()
-        data = data.strip()
-        while data:
-            obj, end = decoder.raw_decode(data, 0)
-            data = data[end:].lstrip()
+        for obj in json_decode_stream(data):
             mail = Mail.from_json(obj)
             mail.queue_id = id_prefix + mail.queue_id
             queue.append(mail)
@@ -222,6 +233,8 @@ class Mail:
 
 # abstract collection/cluster of postfix nodes (or just a single one)
 class Source(abc.ABC):
+    __slots__ = ()
+
     # list of server names in collection
     @abc.abstractmethod
     def server_list(self) -> list[str]:
@@ -320,6 +333,8 @@ class Source(abc.ABC):
 
 
 class LocalSource(Source):
+    __slots__ = ()
+
     def server_list(self) -> list[str]:
         return [socket.gethostname()]
 
@@ -377,6 +392,8 @@ class LocalSource(Source):
 
 
 class RemoteSource(Source):
+    __slots__ = ('remotes',)
+
     def __init__(self, remotes: dict[str, str]) -> None:
         super().__init__()
         self.remotes = remotes
@@ -509,6 +526,8 @@ class RemoteSource(Source):
 
 
 class UserActivityStats:
+    __slots__ = ('count_mails', 'related_mails', 'cut_off')
+
     # find most active address for `health` command
 
     def __init__(self, cut_off: int = 10) -> None:
@@ -577,6 +596,8 @@ class UserActivityStats:
 class Filter(abc.ABC):
     """abstract base class for filter expressions"""
 
+    __slots__ = ()
+
     def __init__(self) -> None:
         pass
 
@@ -639,11 +660,14 @@ class Filter(abc.ABC):
 
 # abstract base class for filters that don't need (...) around in representations of members in combined filters
 class SingleExprFilter(Filter):
-    pass
+    __slots__ = ()
 
 
 class FalseFilter(SingleExprFilter):
     """constant false - matches no mail"""
+
+    __slots__ = ()
+
     def __repr__(self) -> str:
         return '0'
 
@@ -669,6 +693,9 @@ class FalseFilter(SingleExprFilter):
 
 class TrueFilter(SingleExprFilter):
     """constant true - matches all mail"""
+
+    __slots__ = ()
+
     def __repr__(self) -> str:
         return '1'
 
@@ -697,6 +724,8 @@ TRUE_FILTER = TrueFilter()
 
 
 class AndFilter(Filter):
+    __slots__ = ('expressions',)
+
     def __init__(self, expressions: list[Filter] | tuple[()] = ()) -> None:
         super().__init__()
         self.expressions: list[Filter] = expressions or []
@@ -750,6 +779,8 @@ class AndFilter(Filter):
 
 
 class OrFilter(Filter):
+    __slots__ = ('expressions',)
+
     def __init__(self, expressions: list[Filter] | tuple[()] = ()) -> None:
         super().__init__()
         self.expressions: list[Filter] = expressions or []
@@ -803,6 +834,8 @@ class OrFilter(Filter):
 
 
 class NotFilter(SingleExprFilter):
+    __slots__ = ('expression',)
+
     def __init__(self, expression: Filter) -> None:
         super().__init__()
         self.expression = expression
@@ -840,6 +873,9 @@ class NotFilter(SingleExprFilter):
 
 class QueueFilter(SingleExprFilter):
     """match mails based on queue they are in"""
+
+    __slots__ = ('select',)
+
     def __init__(self, select: list[QueueName]) -> None:
         self.select = set(select)
 
@@ -898,6 +934,8 @@ class AddressSelector(enum.Enum):
 
 
 class BaseAddressPattern(abc.ABC):
+    __slots__ = ()
+
     # abstract base class of patterns to use to match a mail address
     # subclasses match either: full address, domain part, regex
     def __init__(self) -> None:
@@ -930,6 +968,8 @@ class BaseAddressPattern(abc.ABC):
 
 
 class AddressPattern(BaseAddressPattern):
+    __slots__ = ('address',)
+
     # match full address exactly
     def __init__(self, address: str) -> None:
         super().__init__()
@@ -952,6 +992,8 @@ class AddressPattern(BaseAddressPattern):
 
 
 class AddressDomainPattern(BaseAddressPattern):
+    __slots__ = ('domain',)
+
     # match address by domain
     def __init__(self, domain: str) -> None:
         super().__init__()
@@ -969,6 +1011,8 @@ class AddressDomainPattern(BaseAddressPattern):
 
 
 class AddressRegexMatch(BaseAddressPattern):
+    __slots__ = ('address',)
+
     # match address by regex
     def __init__(self, address: str | re.Pattern[str]) -> None:
         super().__init__()
@@ -990,6 +1034,8 @@ class AddressRegexMatch(BaseAddressPattern):
 
 
 class AddressFilter(SingleExprFilter):
+    __slots__ = ('selector', 'patterns')
+
     # match mails by address
     def __init__(self, selector: AddressSelector, patterns: list[BaseAddressPattern]) -> None:
         self.selector = selector