placement_bot/job_parser.py at main · Arjun7A/placement_bot · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
import hashlib
import re
from dataclasses import dataclass
from urllib.parse import parse_qs, urljoin, urlparse

import requests
from bs4 import BeautifulSoup


@dataclass
class JobPosting:
    job_id: str
    title: str
    company: str
    location: str
    job_link: str
    description_text: str = ""


def _clean_text(raw_text: str) -> str:
    return re.sub(r"\s+", " ", raw_text).strip()


def _canonicalize_job_link(job_link: str) -> str:
    parsed_url = urlparse(job_link)
    clean_path = re.sub(r"/+", "/", parsed_url.path).rstrip("/")
    return f"{parsed_url.scheme}://{parsed_url.netloc}{clean_path}"


def _extract_job_id(
    card: BeautifulSoup,
    job_link: str,
    title: str,
    company: str,
    location: str,
) -> str:
    urn_candidates = [
        card.get("data-entity-urn", ""),
        card.get("data-id", ""),
    ]

    for candidate in urn_candidates:
        match = re.search(r"(\d{6,})", candidate or "")
        if match:
            return match.group(1)

    link_match = re.search(r"/view/(?:[^/?#]+-)?(\d{6,})(?:[/?#]|$)", job_link)
    if link_match:
        return link_match.group(1)

    parsed_url = urlparse(job_link)
    current_job_id = parse_qs(parsed_url.query).get("currentJobId")
    if current_job_id:
        return current_job_id[0]

    fallback_source = "|".join(
        [
            _canonicalize_job_link(job_link).lower(),
            _clean_text(title).lower(),
            _clean_text(company).lower(),
            _clean_text(location).lower(),
        ]
    )
    return hashlib.sha1(fallback_source.encode("utf-8")).hexdigest()[:16]


def parse_job_cards(html_chunks: list[str]) -> list[JobPosting]:
    parsed_jobs: list[JobPosting] = []

    for html in html_chunks:
        soup = BeautifulSoup(html, "html.parser")
        cards = soup.select("li")

        for card in cards:
            title_node = card.select_one("h3.base-search-card__title") or card.select_one(
                "h3"
            )
            company_node = card.select_one("h4.base-search-card__subtitle") or card.select_one(
                "h4"
            )
            location_node = card.select_one(
                "span.job-search-card__location"
            ) or card.select_one("span")
            link_node = card.select_one("a.base-card__full-link") or card.select_one("a")

            if not link_node:
                continue

            job_link = (link_node.get("href") or "").strip()
            if not job_link:
                continue
            job_link = urljoin("https://www.linkedin.com", job_link)
            job_link = _canonicalize_job_link(job_link)

            title = _clean_text(title_node.get_text(" ", strip=True) if title_node else "")
            company = _clean_text(
                company_node.get_text(" ", strip=True) if company_node else ""
            )
            location = _clean_text(
                location_node.get_text(" ", strip=True) if location_node else ""
            )

            job = JobPosting(
                job_id=_extract_job_id(
                    card=card,
                    job_link=job_link,
                    title=title,
                    company=company,
                    location=location,
                ),
                title=title,
                company=company,
                location=location,
                job_link=job_link,
            )
            parsed_jobs.append(job)

    return parsed_jobs


def fetch_job_description(
    session: requests.Session, job_link: str, timeout_seconds: int = 20
) -> str:
    response = session.get(job_link, timeout=timeout_seconds)
    response.raise_for_status()

    soup = BeautifulSoup(response.text, "html.parser")
    selectors = [
        "div.show-more-less-html__markup",
        "div.description__text",
        "section.show-more-less-html",
        "div.jobs-box__html-content",
    ]

    description_text = ""
    for selector in selectors:
        description_node = soup.select_one(selector)
        if description_node:
            description_text = description_node.get_text(" ", strip=True)
            if description_text:
                break

    if not description_text:
        description_text = soup.get_text(" ", strip=True)

    return _clean_text(description_text)