fix(auth): reload token file when expiry cannot be determined

docs/distributed_training.md

@@ -111,7 +111,13 @@ contacts the auth server once the token is within `leeway` of expiry.
 
 Whatever the transport, pass the **complete token object** around and keep
 its absolute `expires_at` field — it's how a client decides when a token has
-expired. If you produce tokens without the SDK (e.g. a `curl` call to the
-token endpoint), the response only carries the relative `expires_in`, so add
-`expires_at` yourself. And if your transport is a shared file, write it
-atomically (temp file + `os.replace`) so readers never see a partial token.
+expired. `expires_at` is **Unix epoch seconds** (`time.time()`-style), which
+is timezone-independent: the coordinator and the workers can run in
+different timezones or regions without any conversion. If you produce tokens
+without the SDK (e.g. a `curl` call to the token endpoint), the response
+only carries the relative `expires_in`, so add
+`expires_at = time.time() + expires_in` yourself — as epoch seconds, never a
+formatted datetime string or local wall-clock arithmetic, otherwise clients
+cannot check expiry and keep using the token after it has expired. And if
+your transport is a shared file, write it atomically (temp file +
+`os.replace`) so readers never see a partial token.

openapi/templates/rest.mustache

@@ -111,9 +111,12 @@ class RESTClientObject:
     def _reload_token_from_file_if_expired(self, token_file: str):
         # A pre-supplied token has no refresh path of its own; an external
         # process keeps the token file fresh, so re-read it once ours expires.
+        # is_expired() returns None when expires_at is missing or not epoch
+        # seconds — treat that as expired so a malformed token keeps being
+        # re-read instead of being trusted as never expiring.
         assert self.session is not None
         token = self.session.token
-        if token is not None and not token.is_expired(leeway=self._token_leeway):
+        if token is not None and token.is_expired(leeway=self._token_leeway) is False:
             return
         with open(token_file) as f:
             self.session.token = json.load(f)
@@ -146,6 +149,18 @@ class RESTClientObject:
                 "token_type, and an absolute expires_at timestamp.",
                 ", ".join(missing),
             )
+        else:
+            try:
+                int(token["expires_at"])
+            except (TypeError, ValueError):
+                _logger.warning(
+                    "set_token received a token whose expires_at (%r) is not "
+                    "Unix epoch seconds, so its expiry cannot be checked and "
+                    "it will be treated as never expiring. Produce expires_at "
+                    "with time.time()-style epoch seconds, not a formatted "
+                    "datetime.",
+                    token["expires_at"],
+                )
         self.session.token = token
 
     def request(

src/rapidata/api_client/rest.py

@@ -120,9 +120,12 @@ def setup_oauth_with_token(
     def _reload_token_from_file_if_expired(self, token_file: str):
         # A pre-supplied token has no refresh path of its own; an external
         # process keeps the token file fresh, so re-read it once ours expires.
+        # is_expired() returns None when expires_at is missing or not epoch
+        # seconds — treat that as expired so a malformed token keeps being
+        # re-read instead of being trusted as never expiring.
         assert self.session is not None
         token = self.session.token
-        if token is not None and not token.is_expired(leeway=self._token_leeway):
+        if token is not None and token.is_expired(leeway=self._token_leeway) is False:
             return
         with open(token_file) as f:
             self.session.token = json.load(f)
@@ -155,6 +158,18 @@ def set_token(self, token: dict):
                 "token_type, and an absolute expires_at timestamp.",
                 ", ".join(missing),
             )
+        else:
+            try:
+                int(token["expires_at"])
+            except (TypeError, ValueError):
+                _logger.warning(
+                    "set_token received a token whose expires_at (%r) is not "
+                    "Unix epoch seconds, so its expiry cannot be checked and "
+                    "it will be treated as never expiring. Produce expires_at "
+                    "with time.time()-style epoch seconds, not a formatted "
+                    "datetime.",
+                    token["expires_at"],
+                )
         self.session.token = token
 
     def request(