composer create-project laravel/laravel:^10.3.3 web-scrapping-laravel-10
To enable maintenance mode in Laravel, you can use the down command. Open your terminal and navigate to your Laravel project directory. Then, run the following command: php artisan down
To disable maintenance mode, use the up command: php artisan up
php artisan make:controller ScrapeController php artisan make:model Product php artisan make:provider ScrapeServiceProvider
composer require symfony/css-selector composer require guzzlehttp/guzzle composer require symfony/dom-crawler
php artisan make:command ScrapeProducts
php artisan scrape:run
php artisan serve http://localhost:8000/scrape
{
"messages": [
"Scrapping completed for website: magpie_website and file has been successfully written to the public.",
"Scrapping completed for website: another_website and file has been successfully written to the public."
]
}
magpie_website output:
[
{
"title": "iPhone 11 Pro 64GB",
"price": 799.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/iphone-11-pro.png",
"capacityMB": 64000,
"colour": "Green",
"availabilityText": "Out of Stock",
"isAvailable": false,
"shippingText": null,
"shippingDate": null
},
{
"title": "iPhone 11 64GB",
"price": 699.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/iphone-11.png",
"capacityMB": 64000,
"colour": "Black",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Unavailable for delivery",
"shippingDate": null
},
{
"title": "iPhone 11 64GB",
"price": 699.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/iphone-11.png",
"capacityMB": 64000,
"colour": " White",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Unavailable for delivery",
"shippingDate": null
},
{
"title": "iPhone 12 Pro Max 128GB",
"price": 1099.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/iphone-12-pro.png",
"capacityMB": 128000,
"colour": "Sky Blue",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Delivery by Thursday 5th Dec 2024",
"shippingDate": "2024-12-05"
},
{
"title": "Samsung Galaxy S20 128GB",
"price": 849.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/s-20.png",
"capacityMB": 128000,
"colour": "Grey",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Available on 11 Dec 2024",
"shippingDate": "2024-12-11"
},
{
"title": "Huawei P30 Pro 128GB",
"price": 699.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/p-30.png",
"capacityMB": 128000,
"colour": "Black",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Free Shipping",
"shippingDate": null
},
{
"title": "Nokia 3310 100MB",
"price": 99.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/nokia-3310.png",
"capacityMB": 100,
"colour": "Orange",
"availabilityText": "Out of Stock",
"isAvailable": false,
"shippingText": "Delivery from Friday 3rd Jan 2025",
"shippingDate": "2025-01-03"
},
{
"title": "Nokia 3310 100MB",
"price": 99.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/nokia-3310.png",
"capacityMB": 100,
"colour": " Yellow",
"availabilityText": "Out of Stock",
"isAvailable": false,
"shippingText": "Delivery from Friday 3rd Jan 2025",
"shippingDate": "2025-01-03"
},
{
"title": "Nokia 3310 100MB",
"price": 99.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/nokia-3310.png",
"capacityMB": 100,
"colour": " Grey",
"availabilityText": "Out of Stock",
"isAvailable": false,
"shippingText": "Delivery from Friday 3rd Jan 2025",
"shippingDate": "2025-01-03"
},
{
"title": "Nokia 3310 100MB",
"price": 99.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/nokia-3310.png",
"capacityMB": 100,
"colour": " Blue",
"availabilityText": "Out of Stock",
"isAvailable": false,
"shippingText": "Delivery from Friday 3rd Jan 2025",
"shippingDate": "2025-01-03"
},
{
"title": "Google Pixel 5 256GB",
"price": 749.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/pixel-5.png",
"capacityMB": 256000,
"colour": "Black",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Delivers 4 Dec 2024",
"shippingDate": "2024-12-04"
},
{
"title": "Sony Xperia 10 32GB",
"price": 449.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/sony-xperia.png",
"capacityMB": 32000,
"colour": "Black",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Free Shipping",
"shippingDate": null
},
{
"title": "Samsung Galaxy Flip 128GB",
"price": 1699.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/flip.png",
"capacityMB": 128000,
"colour": "Slate Grey",
"availabilityText": "Out of Stock",
"isAvailable": false,
"shippingText": null,
"shippingDate": null
},
{
"title": "Samsung Galaxy S20 128GB",
"price": 849.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/s-20.png",
"capacityMB": 128000,
"colour": " Black",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Free Delivery 5 Dec 2024",
"shippingDate": "2024-12-05"
},
{
"title": "Oppo Reno4 Pro 5G 64GB",
"price": 749.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/oppo-reno.png",
"capacityMB": 64000,
"colour": "Blue",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Free Delivery",
"shippingDate": null
},
{
"title": "Huawei P Smart 64GB",
"price": 399.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/huawei-p-smart.png",
"capacityMB": 64000,
"colour": "Black",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Free Delivery tomorrow",
"shippingDate": null
},
{
"title": "Google Pixel 4 64GB",
"price": 499.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/pixel-5.png",
"capacityMB": 64000,
"colour": "Black",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Delivers 04 Dec 2024",
"shippingDate": "2024-12-04"
},
{
"title": "Google Pixel 4 64GB",
"price": 499.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/pixel-5.png",
"capacityMB": 64000,
"colour": " White",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Delivers 04 Dec 2024",
"shippingDate": "2024-12-04"
},
{
"title": "LG K42 128GB",
"price": 499.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/lg-k42.png",
"capacityMB": 128000,
"colour": "Black",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Order within 6 hours and have it 2024-12-06",
"shippingDate": "2024-12-06"
},
{
"title": "Samsung Galaxy S20 64GB",
"price": 799.99,
"imageUrl": "https://www.magpiehq.com/developer-challenge/images/s-20.png",
"capacityMB": 64000,
"colour": "Grey",
"availabilityText": "In Stock",
"isAvailable": true,
"shippingText": "Free Delivery Thursday 5th Dec 2024",
"shippingDate": "2024-12-05"
}
]