User-agent: *
Disallow: /api/userEvents
Disallow: /nfpage/
Disallow: /ur/nfpage/
Disallow: /ads/
Disallow: /international/
Disallow: /*?*load_all_prop=*
Disallow: /recommended_properties*
Disallow: /ur/s/*
Disallow: /news/page/*?utcda-gets-active-plans-to-restart-development-work-in-various-sectors_html*
Disallow: /account
Disallow: /ur/account
Disallow: /captchaChallenge
Disallow: /ur/captchaChallenge

# Disallow crawling relative links for the most populous cities in Pakistan
Disallow: /Attock*
Disallow: /Badin*
Disallow: /Bahawalnagar*
Disallow: /Bahawalpur*
Disallow: /Bhakkar*
Disallow: /Bhalwal*
Disallow: /Burewala*
Disallow: /Chakwal*
Disallow: /Charsadda*
Disallow: /Chiniot*
Disallow: /Chishtian*
Disallow: /Dadu*
Disallow: /Daharki*
Disallow: /Daska*
Disallow: /Faisalabad*
Disallow: /Ghotki*
Disallow: /Gojra*
Disallow: /Gujranwala*
Disallow: /Gujrat*
Disallow: /Hafizabad*
Disallow: /Haroonabad*
Disallow: /Hasilpur*
Disallow: /Hub*
Disallow: /Hyderabad*
Disallow: /Islamabad*
Disallow: /Jacobabad*
Disallow: /Jhang*
Disallow: /Jhelum*
Disallow: /Kamalia*
Disallow: /Karachi*
Disallow: /Kasur*
Disallow: /Khairpur*
Disallow: /Khanewal*
Disallow: /Khanpur*
Disallow: /Khushab*
Disallow: /Khuzdar*
Disallow: /Kohat*
Disallow: /Lahore*
Disallow: /Larkana*
Disallow: /Layyah*
Disallow: /Lodhran*
Disallow: /Mansehra*
Disallow: /Mardan*
Disallow: /Mianwali*
Disallow: /Multan*
Disallow: /Muridke*
Disallow: /Muzaffargarh*
Disallow: /Narowal*
Disallow: /Nawabshah*
Disallow: /Nowshera*
Disallow: /Okara*
Disallow: /Pakpattan*
Disallow: /Peshawar*
Disallow: /Quetta*
Disallow: /Rawalpindi*
Disallow: /Sadiqabad*
Disallow: /Sahiwal*
Disallow: /Samundri*
Disallow: /Sargodha*
Disallow: /Sheikhupura*
Disallow: /Shikarpur*
Disallow: /Sialkot*
Disallow: /Sukkur*
Disallow: /Swabi*
Disallow: /Taxila*
Disallow: /Turbat*
Disallow: /Vehari*
Disallow: /Wazirabad*

# Disallow crawling relative links for other locations
Disallow: /Abbottabad*
Disallow: /Ahmedpur_East*
Disallow: /Arifwala*
Disallow: /Astore*
Disallow: /Awaran*
Disallow: /Bagh*
Disallow: /Bahria*
Disallow: /Bannu*
Disallow: /Bhimber*
Disallow: /Buner*
Disallow: /Chaghi*
Disallow: /Chichawatni*
Disallow: /Chitral*
Disallow: /Chunian*
Disallow: /Depalpur*
Disallow: /Dera*
Disallow: /Dijkot*
Disallow: /Fateh_Jang*
Disallow: /G_Magnolia_Park*
Disallow: /Galyat*
Disallow: /Gharo*
Disallow: /Gilgit*
Disallow: /Gujar_Khan*
Disallow: /Gwadar*
Disallow: /Hangu*
Disallow: /Harappa*
Disallow: /Haripur*
Disallow: /Hasan_Abdal*
Disallow: /Hassan_Abdal*
Disallow: /Hunza*
Disallow: /IEP_Engineers_Town_IEP_Engineers_Town___Sector_A*
Disallow: /Jahanian*
Disallow: /Jalalpur_Jattan*
Disallow: /Jampur*
Disallow: /Jauharabad*
Disallow: /Kahror*
Disallow: /Kalat*
Disallow: /Kharian*
Disallow: /Kotli*
Disallow: /Lalamusa*
Disallow: /Lasbela*
Disallow: /Liaquatpur*
Disallow: /Loralai*
Disallow: /Lower_Dir*
Disallow: /Mailsi*
Disallow: /Makran*
Disallow: /Malakand*
Disallow: /Mandi_Bahauddin*
Disallow: /Matiari*
Disallow: /Mian_Channu*
Disallow: /Mirpur*
Disallow: /Murree*
Disallow: /Muzaffarabad*
Disallow: /Nankana_Sahib*
Disallow: /Naran*
Disallow: /Nasirabad*
Disallow: /Naushahro_Feroze*
Disallow: /Neelum*
Disallow: /Others*
Disallow: /Pind_Dadan_Khan*
Disallow: /Rahim_Yar_Khan*
Disallow: /Rajanpur*
Disallow: /Rawalakot*
Disallow: /Rohri*
Disallow: /Sanghar*
Disallow: /Shahkot*
Disallow: /Shorkot*
Disallow: /Sibi*
Disallow: /Skardu*
Disallow: /Sudhnoti*
Disallow: /Swat*
Disallow: /Tando*
Disallow: /Tharparkar*
Disallow: /Thatta*
Disallow: /Toba_Tek_Singh*
Disallow: /Wah*
Disallow: /Waziristan*
Disallow: /Yazman*
Disallow: /Zhob*

Disallow: /A_Block-6363
Disallow: /Abdul_Hakim-10594
Disallow: /Al_Badar_Town-5092
Disallow: /Al_Fateh_Town-4705
Disallow: /Al_Hafiz_Town-5093
Disallow: /Al_Mujeeb_Town-5094
Disallow: /Al_Mustafa_Town-5095
Disallow: /Al_Najaf_Colony-4902
Disallow: /Al_Raza_Town-4706
Disallow: /Al_Rehman_City-4707
Disallow: /Alipur-10242
Disallow: /Azad_Kashmir-961
Disallow: /Balochistan-1524
Disallow: /Bela-7903
Disallow: /DHA_Defence_Phase_1_DHA_Phase_1___Sector_F_DHA_Phase_1___Defence_Villas-3075
Disallow: /D_Type_Colony-4930
Disallow: /Duniya_Pur-8474
Disallow: /E_Block-6375
Disallow: /FATA-1737
Disallow: /FATA_Others-1738
Disallow: /Faisal_Gardens-4934
Disallow: /Fortress_Gujranwala-9894
Disallow: /Gaarho-10894
Disallow: /Gadoon-11915
Disallow: /H_Block-6379
Disallow: /Haveli_Lakha-10402
Disallow: /Hyder_Bux_Jatoi-6481
Disallow: /Kaghan-9202
Disallow: /Kamoki-10346
Disallow: /Karak-9484
Disallow: /Khipro-12390
Disallow: /Khyber_Pakhtunkhwa-1525
Disallow: /Kot_Addu-9749
Disallow: /Lakki_Marwat-10205
Disallow: /Pakistan-1521
Disallow: /Pindi_Bhattian-975
Disallow: /Pir_Mahal-9508
Disallow: /Pir_Mahal_UK_Garden-9509
Disallow: /Punjab-1522
Disallow: /Ratwal-9027
Disallow: /Rawalkot-976
Disallow: /Sehwan-8607
Disallow: /Shahdadpur-9029
Disallow: /Shakargarh-12170
Disallow: /Sindh-1523
Disallow: /Talagang-12137
Disallow: /Toba_Pirmahal_Road-6359

Disallow: /adserver/
Disallow: /cp/
Disallow: /search/
Disallow: /ur/search/
Disallow: /zodsite/
Disallow: /body/common/listings_nb_places.php
Disallow: /enem.html
Disallow: /extra/
Disallow: /property_listings/
Disallow: /agc_site/
Disallow: /indices/
Disallow: /uae/
Disallow: /br_index/
Disallow: /questionnaire/
Disallow: /indices.html
Disallow: /?dqs=
Disallow: /newsletter_new/
Disallow: /zforum/
Disallow: /31946216/*
Disallow: /tools/area-guides

User-agent: 008
Disallow: /

User-agent: Mediapartners-Google
Allow: /search/

User-agent: Slurp
Crawl-delay: 300

# Spiders added on date 2014-06-17
# Baidu
# http://www.baidu.com/search/spider.html
User-agent: Baiduspider
Disallow: /

# EasouSpider
# http://www.easou.com/search/spider.html
User-agent: EasouSpider
Disallow: /

# Exabot
# http://www.exabot.com/go/robot
User-agent: Exabot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: trovitbot
Disallow: /

User-agent: trovitBot
Disallow: /

User-agent: Baiduspider+
Disallow: /

User-agent: yandexbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: ZumBot
Disallow: /

User-agent: facebookexternalhit/1.1
Crawl-delay: 15

User-agent: facebookexternalhit/1.0
Crawl-delay: 15

User-agent: facebookexternalhit/*
Crawl-delay: 15