From a5b1ce4726321118b3573112f5022069f920309a Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Thu, 9 Oct 2025 17:59:34 -0400 Subject: [PATCH 1/6] older than iocs downloader --- .../enterprise/indicators_of_compromise.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 3526844..74b4492 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -24,19 +24,23 @@ class IoCDownloaderABC(ABC, Generic[T]): RELATIVE_URL = "/v3/enterprise/ioc" FORMAT: ClassVar[str] - def __init__(self, session: FeedlySession, newer_than: Optional[datetime], stream_id: str): + def __init__(self, session: FeedlySession, newer_than: Optional[datetime], older_than: Optional[datetime], stream_id: str): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). The IoCs will be returned along with their context and relationships in a dictionary representing a valid STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. + Use the older_than parameter to filter articles that are older than your last call. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param older_than: Only articles older than this parameter will be used. If None only one call will be make, + and the continuation will be ignored """ self.newer_than = newer_than + self.older_than = older_than self.session = session self.stream_id = stream_id @@ -46,17 +50,22 @@ def download_all(self) -> List[T]: def stream_bundles(self) -> Iterable[T]: continuation = None while True: + params = { + "continuation": continuation, + "streamId": self.stream_id, + "format": self.FORMAT, + } + if self.newer_than: + params["newerThan"] = int(self.newer_than.timestamp()) + if self.older_than: + params["olderThan"] = int(self.older_than.timestamp()) + resp = self.session.make_api_request( f"{self.RELATIVE_URL}", - params={ - "newerThan": int(self.newer_than.timestamp()) if self.newer_than else None, - "continuation": continuation, - "streamId": self.stream_id, - "format": self.FORMAT, - }, + params=params, ) yield self._parse_response(resp) - if not self.newer_than or "link" not in resp.headers: + if (not self.newer_than and not self.older_than) or "link" not in resp.headers: return next_url = resp.headers["link"][1:].split(">")[0] continuation = parse_qs(next_url)["continuation"][0] @@ -70,21 +79,25 @@ def _merge(self, resp_jsons: Iterable[T]) -> T: class IoCDownloaderBuilder: - def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Optional[datetime] = None): + def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Optional[datetime] = None, older_than: Optional[datetime] = None): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). The IoCs will be returned along with their context and relationships in a dictionary representing a valid STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. + Use the older_than parameter to filter articles that are older than your last call. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param older_than: Only articles older than this parameter will be used. If None only one call will be make, + and the continuation will be ignored """ self.session = session self.format = format self.newer_than = newer_than + self.older_than = older_than self.session.api_host = "https://cloud.feedly.com" self.user = self.session.user @@ -113,7 +126,7 @@ def from_stream_id(self, stream_id: str) -> IoCDownloaderABC: IoCFormat.STIX: StixIoCDownloader, IoCFormat.CSV: CsvIoCDownloader, } - return format2class[self.format](session=self.session, newer_than=self.newer_than, stream_id=stream_id) + return format2class[self.format](session=self.session, newer_than=self.newer_than, older_than=self.older_than, stream_id=stream_id) class StixIoCDownloader(IoCDownloaderABC[Dict]): From 63cfaf63eca9e18f9df00ecd4c69b98fe3044986 Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Thu, 9 Oct 2025 18:04:05 -0400 Subject: [PATCH 2/6] formatting --- .../enterprise/indicators_of_compromise.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 74b4492..835158f 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -4,9 +4,9 @@ from datetime import datetime from enum import Enum from itertools import chain +from requests import Response from typing import ClassVar, Dict, Generic, Iterable, List, Optional, TypeVar from urllib.parse import parse_qs -from requests import Response from feedly.api_client.data import Streamable from feedly.api_client.session import FeedlySession @@ -24,7 +24,13 @@ class IoCDownloaderABC(ABC, Generic[T]): RELATIVE_URL = "/v3/enterprise/ioc" FORMAT: ClassVar[str] - def __init__(self, session: FeedlySession, newer_than: Optional[datetime], older_than: Optional[datetime], stream_id: str): + def __init__( + self, + session: FeedlySession, + newer_than: Optional[datetime], + older_than: Optional[datetime], + stream_id: str, + ): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). @@ -60,12 +66,11 @@ def stream_bundles(self) -> Iterable[T]: if self.older_than: params["olderThan"] = int(self.older_than.timestamp()) - resp = self.session.make_api_request( - f"{self.RELATIVE_URL}", - params=params, - ) + resp = self.session.make_api_request(f"{self.RELATIVE_URL}", params=params) yield self._parse_response(resp) - if (not self.newer_than and not self.older_than) or "link" not in resp.headers: + if ( + not self.newer_than and not self.older_than + ) or "link" not in resp.headers: return next_url = resp.headers["link"][1:].split(">")[0] continuation = parse_qs(next_url)["continuation"][0] @@ -126,7 +131,12 @@ def from_stream_id(self, stream_id: str) -> IoCDownloaderABC: IoCFormat.STIX: StixIoCDownloader, IoCFormat.CSV: CsvIoCDownloader, } - return format2class[self.format](session=self.session, newer_than=self.newer_than, older_than=self.older_than, stream_id=stream_id) + return format2class[self.format]( + session=self.session, + newer_than=self.newer_than, + older_than=self.older_than, + stream_id=stream_id, + ) class StixIoCDownloader(IoCDownloaderABC[Dict]): From b3f365bdf7540e4eecae5b4c140a63b1aff1820c Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Thu, 9 Oct 2025 18:52:12 -0400 Subject: [PATCH 3/6] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1fa7f3f..bc25cfd 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ EMAIL = "ml@feedly.com" AUTHOR = "Feedly" REQUIRES_PYTHON = ">=3.6.0" -VERSION = "0.26" +VERSION = "0.27" # What packages are required for this module to be executed? with open("requirements.txt") as f: From ecec1681652c03215ba216d909c761ff9c007165 Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Thu, 9 Oct 2025 19:38:47 -0400 Subject: [PATCH 4/6] add max batches as well --- .../enterprise/indicators_of_compromise.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 835158f..73691c9 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -30,6 +30,7 @@ def __init__( newer_than: Optional[datetime], older_than: Optional[datetime], stream_id: str, + max_batches: Optional[int] = None, ): """ Use this class to export the contextualized IoCs from a stream. @@ -38,23 +39,27 @@ def __init__( STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. Use the older_than parameter to filter articles that are older than your last call. + Use the max_batches parameter to limit the number of batches/pages to retrieve. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored :param older_than: Only articles older than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param max_batches: Maximum number of batches to retrieve. If None, will continue until no more data is available """ self.newer_than = newer_than self.older_than = older_than self.session = session self.stream_id = stream_id + self.max_batches = max_batches def download_all(self) -> List[T]: return self._merge(self.stream_bundles()) def stream_bundles(self) -> Iterable[T]: continuation = None + batch_count = 0 while True: params = { "continuation": continuation, @@ -68,6 +73,12 @@ def stream_bundles(self) -> Iterable[T]: resp = self.session.make_api_request(f"{self.RELATIVE_URL}", params=params) yield self._parse_response(resp) + batch_count += 1 + + # Check if we've reached max_batches limit + if self.max_batches and batch_count >= self.max_batches: + return + if ( not self.newer_than and not self.older_than ) or "link" not in resp.headers: @@ -84,7 +95,14 @@ def _merge(self, resp_jsons: Iterable[T]) -> T: class IoCDownloaderBuilder: - def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Optional[datetime] = None, older_than: Optional[datetime] = None): + def __init__( + self, + session: FeedlySession, + format: IoCFormat, + newer_than: Optional[datetime] = None, + older_than: Optional[datetime] = None, + max_batches: Optional[int] = None, + ): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). @@ -92,17 +110,20 @@ def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Option STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. Use the older_than parameter to filter articles that are older than your last call. + Use the max_batches parameter to limit the number of batches/pages to retrieve. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored :param older_than: Only articles older than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param max_batches: Maximum number of batches to retrieve. If None, will continue until no more data is available """ self.session = session self.format = format self.newer_than = newer_than self.older_than = older_than + self.max_batches = max_batches self.session.api_host = "https://cloud.feedly.com" self.user = self.session.user @@ -136,6 +157,7 @@ def from_stream_id(self, stream_id: str) -> IoCDownloaderABC: newer_than=self.newer_than, older_than=self.older_than, stream_id=stream_id, + max_batches=self.max_batches, ) From 25a9ccb3f259849a2943bb05ba0956c1e77e4bc5 Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Fri, 10 Oct 2025 17:16:27 -0400 Subject: [PATCH 5/6] pr review --- feedly/api_client/enterprise/indicators_of_compromise.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 73691c9..8d2484d 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -79,9 +79,7 @@ def stream_bundles(self) -> Iterable[T]: if self.max_batches and batch_count >= self.max_batches: return - if ( - not self.newer_than and not self.older_than - ) or "link" not in resp.headers: + if not self.newer_than or "link" not in resp.headers: return next_url = resp.headers["link"][1:].split(">")[0] continuation = parse_qs(next_url)["continuation"][0] From 46642111cd6bfac65a5dca78c389f20d7c5a7ad7 Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Fri, 10 Oct 2025 17:52:40 -0400 Subject: [PATCH 6/6] publish yml --- .github/workflows/publish.yml | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..d0976cb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,40 @@ +name: Publish to PyPI + +on: + push: + tags: + - 'v*' # Trigger on version tags like v1.0.0 + release: + types: [published] # Also trigger on GitHub releases + workflow_dispatch: # Allow manual triggering + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + environment: + name: pypi + url: https://pypi.org/project/feedly-client/ + + permissions: + id-token: write # IMPORTANT: OIDC token generation + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + + - name: Build package + run: python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 +