diff --git a/docs/distributed_training.md b/docs/distributed_training.md index f8ce725618..9573119126 100644 --- a/docs/distributed_training.md +++ b/docs/distributed_training.md @@ -111,7 +111,13 @@ contacts the auth server once the token is within `leeway` of expiry. Whatever the transport, pass the **complete token object** around and keep its absolute `expires_at` field — it's how a client decides when a token has -expired. If you produce tokens without the SDK (e.g. a `curl` call to the -token endpoint), the response only carries the relative `expires_in`, so add -`expires_at` yourself. And if your transport is a shared file, write it -atomically (temp file + `os.replace`) so readers never see a partial token. +expired. `expires_at` is **Unix epoch seconds** (`time.time()`-style), which +is timezone-independent: the coordinator and the workers can run in +different timezones or regions without any conversion. If you produce tokens +without the SDK (e.g. a `curl` call to the token endpoint), the response +only carries the relative `expires_in`, so add +`expires_at = time.time() + expires_in` yourself — as epoch seconds, never a +formatted datetime string or local wall-clock arithmetic, otherwise clients +cannot check expiry and keep using the token after it has expired. And if +your transport is a shared file, write it atomically (temp file + +`os.replace`) so readers never see a partial token. diff --git a/openapi/templates/rest.mustache b/openapi/templates/rest.mustache index a9fbb1b240..6b23e1811d 100644 --- a/openapi/templates/rest.mustache +++ b/openapi/templates/rest.mustache @@ -111,9 +111,12 @@ class RESTClientObject: def _reload_token_from_file_if_expired(self, token_file: str): # A pre-supplied token has no refresh path of its own; an external # process keeps the token file fresh, so re-read it once ours expires. + # is_expired() returns None when expires_at is missing or not epoch + # seconds — treat that as expired so a malformed token keeps being + # re-read instead of being trusted as never expiring. assert self.session is not None token = self.session.token - if token is not None and not token.is_expired(leeway=self._token_leeway): + if token is not None and token.is_expired(leeway=self._token_leeway) is False: return with open(token_file) as f: self.session.token = json.load(f) @@ -146,6 +149,18 @@ class RESTClientObject: "token_type, and an absolute expires_at timestamp.", ", ".join(missing), ) + else: + try: + int(token["expires_at"]) + except (TypeError, ValueError): + _logger.warning( + "set_token received a token whose expires_at (%r) is not " + "Unix epoch seconds, so its expiry cannot be checked and " + "it will be treated as never expiring. Produce expires_at " + "with time.time()-style epoch seconds, not a formatted " + "datetime.", + token["expires_at"], + ) self.session.token = token def request( diff --git a/src/rapidata/api_client/rest.py b/src/rapidata/api_client/rest.py index 1f14f8e8ff..d7789df18c 100644 --- a/src/rapidata/api_client/rest.py +++ b/src/rapidata/api_client/rest.py @@ -120,9 +120,12 @@ def setup_oauth_with_token( def _reload_token_from_file_if_expired(self, token_file: str): # A pre-supplied token has no refresh path of its own; an external # process keeps the token file fresh, so re-read it once ours expires. + # is_expired() returns None when expires_at is missing or not epoch + # seconds — treat that as expired so a malformed token keeps being + # re-read instead of being trusted as never expiring. assert self.session is not None token = self.session.token - if token is not None and not token.is_expired(leeway=self._token_leeway): + if token is not None and token.is_expired(leeway=self._token_leeway) is False: return with open(token_file) as f: self.session.token = json.load(f) @@ -155,6 +158,18 @@ def set_token(self, token: dict): "token_type, and an absolute expires_at timestamp.", ", ".join(missing), ) + else: + try: + int(token["expires_at"]) + except (TypeError, ValueError): + _logger.warning( + "set_token received a token whose expires_at (%r) is not " + "Unix epoch seconds, so its expiry cannot be checked and " + "it will be treated as never expiring. Produce expires_at " + "with time.time()-style epoch seconds, not a formatted " + "datetime.", + token["expires_at"], + ) self.session.token = token def request(