Concat warning/LinkedIn error message fix

src/jobspy/__init__.py

@@ -152,9 +152,15 @@ def worker(site):
             jobs_dfs.append(job_df)
 
     if jobs_dfs:
-        jobs_df = pd.concat(jobs_dfs, ignore_index=True)
-        desired_order: list[str] = [
-            "job_url_hyper" if hyperlinks else "job_url",
+        # Step 1: Filter out all-NA columns from each DataFrame before concatenation
+        filtered_dfs = [df.dropna(axis=1, how='all') for df in jobs_dfs]
+
+        # Step 2: Concatenate the filtered DataFrames
+        jobs_df = pd.concat(filtered_dfs, ignore_index=True)
+
+        # Desired column order
+        desired_order = [
+            "job_url_hyper" if 'hyperlinks' in locals() or 'hyperlinks' in globals() else "job_url",
             "site",
             "title",
             "company",
@@ -172,6 +178,16 @@ def worker(site):
             "emails",
             "description",
         ]
-        return jobs_df[desired_order].sort_values(by=['site', 'date_posted'], ascending=[True, False])
+
+        # Step 3: Ensure all desired columns are present, adding missing ones as empty
+        for column in desired_order:
+            if column not in jobs_df.columns:
+                jobs_df[column] = None  # Add missing columns as empty
+
+        # Reorder the DataFrame according to the desired order
+        jobs_df = jobs_df[desired_order]
+
+        # Step 4: Sort the DataFrame as required
+        return jobs_df.sort_values(by=['site', 'date_posted'], ascending=[True, False])
     else:
         return pd.DataFrame()

src/jobspy/scrapers/linkedin/__init__.py

@@ -104,9 +104,9 @@ def scrape(self, scraper_input: ScraperInput) -> JobResponse:
                     return JobResponse(job_list=job_list)
             except Exception as e:
                 if "Proxy responded with" in str(e):
-                    logger.error(f'Indeed: Bad proxy')
+                    logger.error(f'LinkedIn: Bad proxy')
                 else:
-                    logger.error(f'Indeed: {str(e)}')
+                    logger.error(f'LinkedIn: {str(e)}')
                 return JobResponse(job_list=job_list)
 
             soup = BeautifulSoup(response.text, "html.parser")