From 88134662383162c8f4c28b0ec830f8936d560cb6 Mon Sep 17 00:00:00 2001 From: AlcibiadesCleinias Date: Wed, 24 Jul 2024 17:16:53 +0200 Subject: [PATCH 1/7] bump gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 617a5fe..d2c2451 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ leetcode.apkg .mypy_cache .cookies.sh __pycache__ +.idea/ \ No newline at end of file From f6d6581b9ee6c33f115ae02f2e4e7dda1b56ee63 Mon Sep 17 00:00:00 2001 From: AlcibiadesCleinias Date: Wed, 24 Jul 2024 17:24:05 +0200 Subject: [PATCH 2/7] chore: update .gitignore fix: optioanlly use csrf_token from env feat: add generate-with-last-submissions command docs: introdce new method refactor: improve code style a bit --- Makefile | 13 ++- README.md | 20 +++- generate.py | 53 +++++++-- leetcode_anki/helpers/leetcode.py | 177 +++++++++++++++++++++++++++--- 4 files changed, 235 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 1a41739..772fa64 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,18 @@ -generate: +base: # You have to set the variables below in order to # authenticate on leetcode. It is required to read # the information about the problems test ! "x${VIRTUAL_ENV}" = "x" || (echo "Need to run inside venv" && exit 1) pip install -r requirements.txt + +generate: base ## Generate cards without user submission but for all problems available. python3 generate.py + @echo "\033[0;32mSuccess! Now you can import leetcode.apkg to Anki.\033[0m" + +generate-with-last-submissions: base ## Generate cards with user last submissions for only solved problems + python3 generate.py --problem-status AC --include-last-submission True --stop 1 + @echo "\033[0;32mSuccess! Now you can import leetcode.apkg to Anki.\033[0m" + +help: ## List makefile targets + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 7480f90..a7ec5fc 100644 --- a/README.md +++ b/README.md @@ -48,17 +48,24 @@ python -m venv leetcode-anki Then initialize session id variable. You can get it directly from your browser (if you're using chrome, cookies can be found here chrome://settings/cookies/detail?site=leetcode.com) +> Note, since 24.07.24 you need to manually set CSRF token as well. You can find it in the same place as session id. + Linux/Macos ``` export LEETCODE_SESSION_ID="yyy" +export LEETCODE_CSRF_TOKEN="zzz" ``` Windows ``` set LEETCODE_SESSION_ID="yyy" +set LEETCODE_CSRF_TOKEN="zzz" ``` -And finally run for Linux/MacOS +Then you can run the script + +### Classic Lightweight Cards +Run for Linux/MacOS ``` make generate ``` @@ -68,4 +75,15 @@ pip install -r requirements.txt python generate.py ``` +### Including your Last Submission Code +Run for Linux/MacOS +``` +make generate-with-last-submissions +``` +Or for Windows +``` +pip install -r requirements.txt +python generate.py --problem-status AC --include-last-submission True +``` + You'll get `leetcode.apkg` file, which you can import directly to your anki app. diff --git a/generate.py b/generate.py index dafcfc5..6ef0673 100755 --- a/generate.py +++ b/generate.py @@ -1,14 +1,19 @@ #!/usr/bin/env python3 """ -This script generates an Anki deck with all the leetcode problems currently -known. +This script generates an Anki deck +- with all the leetcode problems currently known. + - optionally, with all the leetcode problems that currently have expected status, e.g. submission accepted. +- with the last accepted submission for each problem on back side. + +To work with leetcode API, you need to provide the session id and csrf token (you could find them manually in the browser). """ import argparse import asyncio import logging from pathlib import Path -from typing import Any, Awaitable, Callable, Coroutine, List +from typing import Awaitable, List +import html # https://github.com/kerrickstaley/genanki import genanki # type: ignore @@ -51,6 +56,19 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--output-file", type=str, help="Output filename", default=OUTPUT_FILE ) + parser.add_argument( + "--problem-status", + type=str, + help="Get all problems with specific status {'AC', etc.}", + default="", + ) + parser.add_argument( + "--include-last-submission", + type=bool, + help="Get the last accepted submission for each problem. " + "Note, that this is very heavy operation as it adds 2 additional requests per problem.", + default=False, + ) args = parser.parse_args() @@ -69,6 +87,7 @@ def guid(self) -> str: return genanki.guid_for(self.fields[0]) +# TODO: refactor to separate module. async def generate_anki_note( leetcode_data: leetcode_anki.helpers.leetcode.LeetcodeData, leetcode_model: genanki.Model, @@ -99,15 +118,18 @@ async def generate_anki_note( ) ), str(await leetcode_data.freq_bar(leetcode_task_handle)), + # Use escape to avoid HTML injection. + ("\n" + html.escape(str(await leetcode_data.last_submission_code(leetcode_task_handle))) + if leetcode_data.include_last_submission else ""), ], tags=await leetcode_data.tags(leetcode_task_handle), - # FIXME: sort field doesn't work doesn't work + # FIXME: sort field doesn't work doesn't work (always remember I am patient, I am patient). sort_field=str(await leetcode_data.freq_bar(leetcode_task_handle)).zfill(3), ) async def generate( - start: int, stop: int, page_size: int, list_id: str, output_file: str + start: int, stop: int, page_size: int, list_id: str, output_file: str, problem_status: str, include_last_submission: bool ) -> None: """ Generate an Anki deck @@ -129,6 +151,7 @@ async def generate( {"name": "SubmissionsAccepted"}, {"name": "SumissionAcceptRate"}, {"name": "Frequency"}, + {"name": "LastSubmissionCode"}, # TODO: add hints ], templates=[ @@ -168,7 +191,16 @@ async def generate( https://leetcode.com/problems/{{Slug}}/solution/ -
+ {{#LastSubmissionCode}} +
+ Accepted Last Submission: + 
+                    
+                    {{LastSubmissionCode}}
+                    
+
+
+ {{/LastSubmissionCode}} """, } ], @@ -176,11 +208,12 @@ async def generate( leetcode_deck = genanki.Deck(LEETCODE_ANKI_DECK_ID, Path(output_file).stem) leetcode_data = leetcode_anki.helpers.leetcode.LeetcodeData( - start, stop, page_size, list_id + start, stop, page_size, list_id, problem_status, include_last_submission ) note_generators: List[Awaitable[LeetcodeNote]] = [] + # Fetch all data from Leetcode API. task_handles = await leetcode_data.all_problems_handles() logging.info("Generating flashcards") @@ -201,14 +234,16 @@ async def main() -> None: """ args = parse_args() - start, stop, page_size, list_id, output_file = ( + start, stop, page_size, list_id, output_file, problem_status, include_last_submission = ( args.start, args.stop, args.page_size, args.list_id, args.output_file, + args.problem_status, + args.include_last_submission, ) - await generate(start, stop, page_size, list_id, output_file) + await generate(start, stop, page_size, list_id, output_file, problem_status, include_last_submission) if __name__ == "__main__": diff --git a/leetcode_anki/helpers/leetcode.py b/leetcode_anki/helpers/leetcode.py index f4ba87b..9b33612 100644 --- a/leetcode_anki/helpers/leetcode.py +++ b/leetcode_anki/helpers/leetcode.py @@ -35,7 +35,11 @@ def _get_leetcode_api_client() -> leetcode.api.default_api.DefaultApi: configuration = leetcode.configuration.Configuration() session_id = os.environ["LEETCODE_SESSION_ID"] - csrf_token = leetcode.auth.get_csrf_cookie(session_id) + csrf_token = os.environ.get("LEETCODE_CSRF_TOKEN", None) + # Probably method is deprecated since ~24.07.2024, + # ref to https://github.com/fspv/leetcode-anki/issues/39. + # TODO: check new versions for smooth integration of csrf_cookie. + csrf_token = leetcode.auth.get_csrf_cookie(session_id) if csrf_token is None else csrf_token configuration.api_key["x-csrftoken"] = csrf_token configuration.api_key["csrftoken"] = csrf_token @@ -58,7 +62,7 @@ class _RetryDecorator: _delay: float def __init__( - self, times: int, exceptions: Tuple[Type[Exception]], delay: float + self, times: int, exceptions: Tuple[Type[Exception]], delay: float ) -> None: self._times = times self._exceptions = exceptions @@ -87,7 +91,7 @@ def wrapper(*args: Any, **kwargs: Any) -> _T: def retry( - times: int, exceptions: Tuple[Type[Exception]], delay: float + times: int, exceptions: Tuple[Type[Exception]], delay: float ) -> _RetryDecorator: """ Retry Decorator @@ -105,12 +109,18 @@ class LeetcodeData: This data can be later accessed using provided methods with corresponding names. """ + # Leetcode has a rate limiter. + LEETCODE_API_REQUEST_DELAY = 2 + SUBMISSION_STATUS_ACCEPTED = 10 def __init__( - self, start: int, stop: int, page_size: int = 1000, list_id: str = "" + self, start: int, stop: int, page_size: int = 1000, list_id: str = "", status: str = "", include_last_submission: bool = False ) -> None: """ Initialize leetcode API and disk cache for API responses + @param status: if status is "AC" then only accepted solutions will be fetched. + @param include_last_submission: if True, then last accepted submission will be fetched for each problem. + Note, that this is very heavy operation as it add 2 additional requests per problem. """ if start < 0: raise ValueError(f"Start must be non-negative: {start}") @@ -128,6 +138,8 @@ def __init__( self._stop = stop self._page_size = page_size self._list_id = list_id + self.status = status if status != "" else None + self.include_last_submission = include_last_submission @cached_property def _api_instance(self) -> leetcode.api.default_api.DefaultApi: @@ -135,7 +147,7 @@ def _api_instance(self) -> leetcode.api.default_api.DefaultApi: @cached_property def _cache( - self, + self, ) -> Dict[str, leetcode.models.graphql_question_detail.GraphqlQuestionDetail]: """ Cached method to return dict (problem_slug -> question details) @@ -143,6 +155,16 @@ def _cache( problems = self._get_problems_data() return {problem.title_slug: problem for problem in problems} + @cached_property + def _cache_user_submissions( + self, + ) -> Dict[str, str]: + """ + Cached method to return dict (problem_slug -> last submitted accepted user solution) + """ + problem_to_submission = self._get_submissions_codes_data() + return {problem_slug: code_data for problem_slug, code_data in problem_to_submission.items()} + @retry(times=3, exceptions=(urllib3.exceptions.ProtocolError,), delay=5) def _get_problems_count(self) -> int: api_instance = self._api_instance @@ -166,9 +188,9 @@ def _get_problems_count(self) -> int: skip=0, filters=leetcode.models.graphql_query_problemset_question_list_variables_filter_input.GraphqlQueryProblemsetQuestionListVariablesFilterInput( tags=[], - list_id=self._list_id + list_id=self._list_id, + status=self.status, # difficulty="MEDIUM", - # status="NOT_STARTED", # list_id="7p5x763", # Top Amazon Questions # premium_only=False, ), @@ -176,14 +198,14 @@ def _get_problems_count(self) -> int: operation_name="problemsetQuestionList", ) - time.sleep(2) # Leetcode has a rate limiter + time.sleep(self.LEETCODE_API_REQUEST_DELAY) # Leetcode has a rate limiter data = api_instance.graphql_post(body=graphql_request).data return data.problemset_question_list.total_num or 0 @retry(times=3, exceptions=(urllib3.exceptions.ProtocolError,), delay=5) def _get_problems_data_page( - self, offset: int, page_size: int, page: int + self, offset: int, page_size: int, page: int ) -> List[leetcode.models.graphql_question_detail.GraphqlQuestionDetail]: api_instance = self._api_instance graphql_request = leetcode.models.graphql_query.GraphqlQuery( @@ -221,13 +243,14 @@ def _get_problems_data_page( limit=page_size, skip=offset + page * page_size, filters=leetcode.models.graphql_query_problemset_question_list_variables_filter_input.GraphqlQueryProblemsetQuestionListVariablesFilterInput( - list_id=self._list_id + list_id=self._list_id, + status=self.status, ), ), operation_name="problemsetQuestionList", ) - time.sleep(2) # Leetcode has a rate limiter + time.sleep(self.LEETCODE_API_REQUEST_DELAY) # Leetcode has a rate limiter data = api_instance.graphql_post( body=graphql_request ).data.problemset_question_list.questions @@ -235,7 +258,7 @@ def _get_problems_data_page( return data def _get_problems_data( - self, + self, ) -> List[leetcode.models.graphql_question_detail.GraphqlQuestionDetail]: problem_count = self._get_problems_count() @@ -256,25 +279,139 @@ def _get_problems_data( logging.info("Fetching %s problems %s per page", stop - start + 1, page_size) for page in tqdm( - range(math.ceil((stop - start + 1) / page_size)), - unit="problem", - unit_scale=page_size, + range(math.ceil((stop - start + 1) / page_size)), + unit="problem", + unit_scale=page_size, ): data = self._get_problems_data_page(start, page_size, page) problems.extend(data) return problems + @retry(times=3, exceptions=(urllib3.exceptions.ProtocolError,), delay=5) + def _get_submissions_codes_data(self) -> Dict[str, str]: + """It collects all submissions for the cached problems of the current class object.""" + all_fetched_problems = self._cache.keys() + problem_to_submission: Dict[str, str] = {} + + for problem_slug in tqdm( + all_fetched_problems, + unit="Problem", + ): + logging.info("Fetching submission for problem: %s ", problem_slug) + try: + data = self.get_submission_code(problem_slug) + except Exception as e: + # Log only if submission was expected to be found. + if self.status: + logging.error("Error fetching submission for problem: %s", problem_slug) + logging.exception(e) + data = "" + problem_to_submission[problem_slug] = data + return problem_to_submission + + def get_submission_code(self, problem_slug: str) -> str: + """ + [Experimental feature, 24.07.24] Get user (depends on session cookies) last submitted code + that was accepted for the given problem. + + Note: + - it is sync request. + - it uses 2 raw requests under the hood to leetcode graphQL endpoints. + """ + LIMIT = 500 # Max number of submissions to fetch. + + data = self._api_instance.graphql_post( + body={ + "query": """ + query submissionList($offset: Int!, $limit: Int!, $lastKey: String, $questionSlug: String!, $lang: Int, $status: Int) { + questionSubmissionList( + offset: $offset + limit: $limit + lastKey: $lastKey + questionSlug: $questionSlug + lang: $lang + status: $status + ) { + lastKey + hasNext + submissions { + id + } + } + } + """, + "variables": { + "questionSlug": problem_slug, + "offset": 0, + "limit": LIMIT, + "lastKey": None, + "status": self.SUBMISSION_STATUS_ACCEPTED, + }, + "operationName": "submissionList" + }, + _preload_content=False, # The key to make it works and return raw content. + ) + # Reponse format: {'data': {'questionSubmissionList': + # {'lastKey': None, 'hasNext': False, 'submissions': [{'id': '969483658', <...>}]}}} + payload = data.json() + + # Check that somthing returnd and remember the first id. + accepted_submissions = payload.get("data", {}).get("questionSubmissionList", {}).get("submissions", {}) + if not accepted_submissions: + raise Exception("No accepted submissions found") + first_submission_id = accepted_submissions[0]["id"] + + time.sleep(self.LEETCODE_API_REQUEST_DELAY) + + # Get Submission details (we want to get code part). + data = self._api_instance.graphql_post( + body={ + "query": """ + query submissionDetails($submissionId: Int!) { + submissionDetails(submissionId: $submissionId) { + code + lang { + name + verboseName + } + } + } + """, + "variables": { + "submissionId": first_submission_id + }, + "operationName": "submissionDetails" + }, + _preload_content=False, # The key to make it work and return raw content. + ) + # E.g. repspons: { "data": { "submissionDetails": { <...> "code": "<...>", <...>} } } + payload = data.json() + # Get code if possible. + code = payload.get("data", {}).get("submissionDetails", {}).get("code", "") + if not code: + raise Exception("No code found") + + return code + async def all_problems_handles(self) -> List[str]: """ Get all problem handles known. + This method is used to initiate fetching of all data needed via blocking call. Example: ["two-sum", "three-sum"] """ - return list(self._cache.keys()) + # Fetch problems if not yet fetched. + problem_slugs = list(self._cache.keys()) + + # Fetch submissions if not yet fetched and needed. + if self.include_last_submission: + _ = self._cache_user_submissions + + return problem_slugs def _get_problem_data( - self, problem_slug: str + self, problem_slug: str ) -> leetcode.models.graphql_question_detail.GraphqlQuestionDetail: """ TODO: Legacy method. Needed in the old architecture. Can be replaced @@ -286,6 +423,12 @@ def _get_problem_data( raise ValueError(f"Problem {problem_slug} is not in cache") + async def last_submission_code(self, problem_slug: str) -> str: + """ + Last accepted submission code. + """ + return self._cache_user_submissions.get(problem_slug, "No code found.") + async def _get_description(self, problem_slug: str) -> str: """ Problem description From 8c4f0f93a2b7a3180242a276d8eef13c2e6edd4b Mon Sep 17 00:00:00 2001 From: AlcibiadesCleinias Date: Wed, 24 Jul 2024 17:24:56 +0200 Subject: [PATCH 3/7] - Change the `--stop 1` argument to the `generate.py` script to remove the limit on the number of problems to generate cards for --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 772fa64..732704b 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ generate: base ## Generate cards without user submission but for all problems av @echo "\033[0;32mSuccess! Now you can import leetcode.apkg to Anki.\033[0m" generate-with-last-submissions: base ## Generate cards with user last submissions for only solved problems - python3 generate.py --problem-status AC --include-last-submission True --stop 1 + python3 generate.py --problem-status AC --include-last-submission True @echo "\033[0;32mSuccess! Now you can import leetcode.apkg to Anki.\033[0m" help: ## List makefile targets From b10ea35fe1cf4033c495f3019ef0bd77ee8d3eeb Mon Sep 17 00:00:00 2001 From: AlcibiadesCleinias Date: Wed, 24 Jul 2024 17:27:14 +0200 Subject: [PATCH 4/7] feat: update dependencies in requirements.txt - Upgrade `genanki` to version 0.13.1 - Upgrade `tqdm` to version 4.66.4 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 951466f..a6e0e26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ python-leetcode==1.2.1 setuptools==57.5.0 -genanki -tqdm +genanki==0.13.1 +tqdm==4.66.4 From 9405d9fd87fb3bd57932d1ece2edde08a060d6c7 Mon Sep 17 00:00:00 2001 From: AlcibiadesCleinias Date: Wed, 24 Jul 2024 17:48:53 +0200 Subject: [PATCH 5/7] feat: add details for obtaining CSRF token and example cookie image - Added a new details section with an example image showing where to find the CSRF token in the browser cookies - Added a new details section with an example image showing the format of the last submission code on the back card --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index a7ec5fc..525222c 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ Then initialize session id variable. You can get it directly from your browser ( > Note, since 24.07.24 you need to manually set CSRF token as well. You can find it in the same place as session id. +
+ Chrome example cookie + +
+ Linux/Macos ``` export LEETCODE_SESSION_ID="yyy" @@ -76,6 +81,11 @@ python generate.py ``` ### Including your Last Submission Code +
+ Example if code on the back card part + +
+ Run for Linux/MacOS ``` make generate-with-last-submissions From 2ca9b5a22a522422f962f3b8dbf21b928d88a422 Mon Sep 17 00:00:00 2001 From: AlcibiadesCleinias Date: Wed, 24 Jul 2024 18:06:03 +0200 Subject: [PATCH 6/7] Check if `self.include_last_submission` is True before logging an error when the last submission data is not available --- leetcode_anki/helpers/leetcode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leetcode_anki/helpers/leetcode.py b/leetcode_anki/helpers/leetcode.py index 9b33612..080a039 100644 --- a/leetcode_anki/helpers/leetcode.py +++ b/leetcode_anki/helpers/leetcode.py @@ -303,7 +303,7 @@ def _get_submissions_codes_data(self) -> Dict[str, str]: data = self.get_submission_code(problem_slug) except Exception as e: # Log only if submission was expected to be found. - if self.status: + if self.status and self.include_last_submission: logging.error("Error fetching submission for problem: %s", problem_slug) logging.exception(e) data = "" From f3aa567e7e07f16b223da487aaa4170dfb532443 Mon Sep 17 00:00:00 2001 From: AlcibiadesCleinias Date: Wed, 24 Jul 2024 18:08:40 +0200 Subject: [PATCH 7/7] Clarify that the `all_problems_handles` method is used to initiate fetching of all data needed from Leetcode, and that it is a blocking call. --- leetcode_anki/helpers/leetcode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leetcode_anki/helpers/leetcode.py b/leetcode_anki/helpers/leetcode.py index 080a039..64a08d9 100644 --- a/leetcode_anki/helpers/leetcode.py +++ b/leetcode_anki/helpers/leetcode.py @@ -397,7 +397,7 @@ def get_submission_code(self, problem_slug: str) -> str: async def all_problems_handles(self) -> List[str]: """ Get all problem handles known. - This method is used to initiate fetching of all data needed via blocking call. + This method is used to initiate fetching of all data needed from Leetcode, and via blocking call. Example: ["two-sum", "three-sum"] """