Spaces:
Runtime error
Runtime error
File size: 6,067 Bytes
9e6917b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 |
import requests
import json
from urllib.parse import urljoin
import pymongo
from pymongo import MongoClient
import certifi
from Mongo.Noonfood_Mongo_URL_From_location import Mongo_location_URLS
def mutiple_url_location(lat , lng):
def get_initial_data(lat, lng):
initial_url = "https://food.noon.com/_svc/customer-v1/customer/public-area-serviceable"
initial_payload = {"lat": str(lat), "lng": str(lng)}
initial_headers = {
"Accept": "application/json, text/plain, */*",
"Accept-Encoding": "gzip, deflate, br, zstd",
"Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
"Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
"Content-Type": "application/json",
"Origin": "https://food.noon.com",
"Referer": "https://food.noon.com/",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"X-Content": "desktop",
"X-Experience": "food",
"X-Locale": "en-ae",
"X-Mp": "noon",
"X-Platform": "web",
"X-Visitor-Id": "1072f8c2-cfd6-4734-8795-d637de61fba1",
}
try:
response = requests.post(initial_url, json=initial_payload, headers=initial_headers)
response.raise_for_status()
json_data = response.json()
print(f"Initial Response JSON: {json_data}")
new_lat = json_data.get('lat')
new_lng = json_data.get('lng')
if new_lat and new_lng:
return json_data, new_lat, new_lng
else:
print("lat or lng not found in the initial response.")
return None, None, None
except requests.exceptions.RequestException as e:
print(f"Request failed: {e}")
return None, None, None
def get_urls_from_json(json_data):
url_list = []
try:
results = json_data.get("results", [])
if len(results) > 5:
banners = results[5]['modules'][0].get("banners", [])
for n in range(min(20, len(banners))):
try:
base_url = 'https://food.noon.com/'
want_data = banners[n].get("linkUrl")
if want_data:
full_url = urljoin(base_url, want_data)
url_list.append(full_url)
except Exception as e:
print(f"Error occurred while extracting URLs: {e}")
break
else:
print("Insufficient data in 'results'")
except Exception as e:
print(f"Error parsing JSON data: {e}")
return url_list
def store_data_in_mongo(json_data, url_list):
try:
# MongoDB connection settings
client = MongoClient(
"mongodb+srv://dipenigenerate:[email protected]",
tlsCAFile=certifi.where()
)
db = client['Restaurants_in_dubai']
collection = db['noonfood_link']
# Create the document to insert
document = {
"initial_response": json_data,
"url_list": url_list
}
# Insert the document into the collection
result = collection.insert_one(document)
print(f"Data inserted with id: {result.inserted_id}")
except Exception as e:
print(f"Failed to store data in MongoDB: {e}")
def fetch_urls(lat, lng):
json_data, new_lat, new_lng = get_initial_data(lat, lng)
if new_lat and new_lng:
new_url = "https://food.noon.com/_svc/mp-food-api-catalog/api/"
new_headers = {
"method": "GET",
"Accept": "application/json, text/plain, */*",
"Accept-Encoding": "gzip, deflate, br, zstd",
"Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
"Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
"Content-Type": "application/json",
"Origin": "https://food.noon.com",
"Referer": "https://food.noon.com/",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"X-Content": "desktop",
"X-Experience": "food",
"X-Locale": "en-ae",
"X-Mp": "noon",
"X-Platform": "web",
"X-lat": f"{new_lat}",
"X-lng": f"{new_lng}"
}
try:
new_response = requests.get(new_url, headers=new_headers)
new_response.raise_for_status()
new_json_data = new_response.json()
# Extract URLs from the JSON data
url_list = get_urls_from_json(new_json_data)
# Store the initial JSON response and URLs in MongoDB
store_data_in_mongo(json_data, url_list)
return url_list
except requests.exceptions.RequestException as e:
print(f"Failed to retrieve new content from the URL: {e}")
return []
else:
return []
urls = fetch_urls(lat, lng)
if urls:
print("Fetched URLs:" , urls)
for url in urls:
url = url+"/"
Mongo_location_URLS(url,lat,lng)
# if __name__ == "__main__":
# lat = int(input("Enter the latitude: "))
# lng = int(input("Enter the longitude: "))
# urls = fetch_urls(lat, lng)
# if urls:
# print("Fetched URLs:" , urls)
# for url in urls:
# url = url+"/"
# Mongo_location_URLS(url,lat,lng)
# else:
# print("No URLs fetched.")
|