<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"
	xmlns:content="http://purl.org/rss/1.0/modules/content/"
	xmlns:wfw="http://wellformedweb.org/CommentAPI/"
	xmlns:dc="http://purl.org/dc/elements/1.1/"
	xmlns:atom="http://www.w3.org/2005/Atom"
	xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
	xmlns:slash="http://purl.org/rss/1.0/modules/slash/"
	>

<channel>
	<title>Thaidoweb 2008 &#187; Robots Txt</title>
	<atom:link href="http://www.thaidoweb.com/tag/robots-txt/feed/" rel="self" type="application/rss+xml" />
	<link>http://www.thaidoweb.com</link>
	<description>All About Computers , Internet and Website.</description>
	<lastBuildDate>Sun, 13 Nov 2011 07:26:08 +0000</lastBuildDate>
	<language>en</language>
	<sy:updatePeriod>hourly</sy:updatePeriod>
	<sy:updateFrequency>1</sy:updateFrequency>
	<generator>http://wordpress.org/?v=3.0.1</generator>
		<item>
		<title>How to Prevent Duplicate Content with Effective Use of the Robots.txt and Robots Meta Tag</title>
		<link>http://www.thaidoweb.com/2009/06/30/how-to-prevent-duplicate-content-with-effective-use-of-the-robotstxt-and-robots-meta-tag/</link>
		<comments>http://www.thaidoweb.com/2009/06/30/how-to-prevent-duplicate-content-with-effective-use-of-the-robotstxt-and-robots-meta-tag/#comments</comments>
		<pubDate>Mon, 29 Jun 2009 20:45:03 +0000</pubDate>
		<dc:creator></dc:creator>
				<category><![CDATA[Web Traffic]]></category>
		<category><![CDATA[about SEO]]></category>
		<category><![CDATA[Asterisk]]></category>
		<category><![CDATA[Dynamic Sites]]></category>
		<category><![CDATA[Engine Optimization Services]]></category>
		<category><![CDATA[Fi]]></category>
		<category><![CDATA[How to Prevent Duplicate Content with Effective Use of the Robots.txt and Robots Meta Tag]]></category>
		<category><![CDATA[Images]]></category>
		<category><![CDATA[Major Search Engines]]></category>
		<category><![CDATA[Meta Content]]></category>
		<category><![CDATA[Notepad]]></category>
		<category><![CDATA[Robot Exclusion Protocol]]></category>
		<category><![CDATA[Robots Meta Tag]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[Root Directory]]></category>
		<category><![CDATA[Search Engine Optimization]]></category>
		<category><![CDATA[Search Engine Optimization (SEO) Information]]></category>
		<category><![CDATA[Search Engine Optimization Services]]></category>
		<category><![CDATA[Search Engine Spider]]></category>
		<category><![CDATA[Search Engine Spiders]]></category>
		<category><![CDATA[Search Optimization]]></category>
		<category><![CDATA[SEO]]></category>
		<category><![CDATA[SEO articles]]></category>
		<category><![CDATA[SEO information]]></category>
		<category><![CDATA[Spider Indexes]]></category>
		<category><![CDATA[Two Ways]]></category>
		<category><![CDATA[Weapon Of Choice]]></category>
		<category><![CDATA[what is SEO]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2009/06/30/how-to-prevent-duplicate-content-with-effective-use-of-the-robotstxt-and-robots-meta-tag/</guid>
		<description><![CDATA[Duplicate content is one of the problems that we regularly come across as part of the search engine optimization services we offer. If the search engines determine your site contains similar content, this may result in penalties and even exclusion from the search engines. Fortunately it&#8217;s a problem that is easily rectified. Your primary weapon [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2009/06/30/how-to-prevent-duplicate-content-with-effective-use-of-the-robotstxt-and-robots-meta-tag/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>Search Engine Spiders Lost Without Guidance &#8211; Post This Sign!</title>
		<link>http://www.thaidoweb.com/2009/06/06/search-engine-spiders-lost-without-guidance-post-this-sign/</link>
		<comments>http://www.thaidoweb.com/2009/06/06/search-engine-spiders-lost-without-guidance-post-this-sign/#comments</comments>
		<pubDate>Fri, 05 Jun 2009 23:45:02 +0000</pubDate>
		<dc:creator></dc:creator>
				<category><![CDATA[Web Traffic]]></category>
		<category><![CDATA[about SEO]]></category>
		<category><![CDATA[Asterisk]]></category>
		<category><![CDATA[Benefit]]></category>
		<category><![CDATA[google]]></category>
		<category><![CDATA[Google Crawls]]></category>
		<category><![CDATA[Group User]]></category>
		<category><![CDATA[Guidance]]></category>
		<category><![CDATA[Images]]></category>
		<category><![CDATA[Lost]]></category>
		<category><![CDATA[Msnbot]]></category>
		<category><![CDATA[Plain Text Document]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[Search Engine Optimization (SEO) Information]]></category>
		<category><![CDATA[Search Engine Spiders]]></category>
		<category><![CDATA[Search Engine Spiders Lost Without Guidance - Post This Sign!]]></category>
		<category><![CDATA[SEO]]></category>
		<category><![CDATA[SEO articles]]></category>
		<category><![CDATA[SEO information]]></category>
		<category><![CDATA[Server Resources]]></category>
		<category><![CDATA[Teoma]]></category>
		<category><![CDATA[Tiny Text]]></category>
		<category><![CDATA[Web Crawlers]]></category>
		<category><![CDATA[what is SEO]]></category>
		<category><![CDATA[Wild Card]]></category>
		<category><![CDATA[yahoo]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2009/06/06/search-engine-spiders-lost-without-guidance-post-this-sign/</guid>
		<description><![CDATA[The robots.txt file is an exclusion standard required by all web crawlers/robots to tell them what files and directories that you want them to stay OUT of on your site. Not all crawlers/bots follow the exclusion standard and will continue crawling your site anyway. I like to call them &#8220;Bad Bots&#8221; or trespassers. We block [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2009/06/06/search-engine-spiders-lost-without-guidance-post-this-sign/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>The Role of the Robots.txt File to Improve Site Ranking!</title>
		<link>http://www.thaidoweb.com/2009/06/02/the-role-of-the-robotstxt-file-to-improve-site-ranking/</link>
		<comments>http://www.thaidoweb.com/2009/06/02/the-role-of-the-robotstxt-file-to-improve-site-ranking/#comments</comments>
		<pubDate>Tue, 02 Jun 2009 15:45:02 +0000</pubDate>
		<dc:creator></dc:creator>
				<category><![CDATA[Web Traffic]]></category>
		<category><![CDATA[about SEO]]></category>
		<category><![CDATA[Blank Text]]></category>
		<category><![CDATA[Directories]]></category>
		<category><![CDATA[Folders]]></category>
		<category><![CDATA[Hosting Directory]]></category>
		<category><![CDATA[Improve Ranking]]></category>
		<category><![CDATA[Programming Languages]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[Search Engine Optimization (SEO) Information]]></category>
		<category><![CDATA[Search Engine Spiders]]></category>
		<category><![CDATA[search engines]]></category>
		<category><![CDATA[Search Robots]]></category>
		<category><![CDATA[SEO]]></category>
		<category><![CDATA[SEO articles]]></category>
		<category><![CDATA[SEO information]]></category>
		<category><![CDATA[Seperate Directory]]></category>
		<category><![CDATA[Site Web]]></category>
		<category><![CDATA[The Role of the Robots.txt File to Improve Site Ranking!]]></category>
		<category><![CDATA[Unwanted Files]]></category>
		<category><![CDATA[Upload]]></category>
		<category><![CDATA[Web Content]]></category>
		<category><![CDATA[Web Directory]]></category>
		<category><![CDATA[Web Hosting]]></category>
		<category><![CDATA[Web Master]]></category>
		<category><![CDATA[Web Site Content]]></category>
		<category><![CDATA[Web Stats]]></category>
		<category><![CDATA[what is SEO]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2009/06/02/the-role-of-the-robotstxt-file-to-improve-site-ranking/</guid>
		<description><![CDATA[Not many web master take the time to use a robots.txt file for their website. For search engine spiders that use the robots.txt to see what directories to search through, the robots.txt file can be very helpful in keeping the spiders indexing your actual pages and not other information, such as looking through your stats! [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2009/06/02/the-role-of-the-robotstxt-file-to-improve-site-ranking/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>Link Exchange Tips, No Tricks</title>
		<link>http://www.thaidoweb.com/2009/05/29/link-exchange-tips-no-tricks/</link>
		<comments>http://www.thaidoweb.com/2009/05/29/link-exchange-tips-no-tricks/#comments</comments>
		<pubDate>Fri, 29 May 2009 05:45:03 +0000</pubDate>
		<dc:creator></dc:creator>
				<category><![CDATA[Web Traffic]]></category>
		<category><![CDATA[about SEO]]></category>
		<category><![CDATA[Back Links]]></category>
		<category><![CDATA[Exchange Tips]]></category>
		<category><![CDATA[Exchanging Links]]></category>
		<category><![CDATA[Football League]]></category>
		<category><![CDATA[Href Link]]></category>
		<category><![CDATA[Html Files]]></category>
		<category><![CDATA[Html Tips]]></category>
		<category><![CDATA[Html Title]]></category>
		<category><![CDATA[Html Tricks]]></category>
		<category><![CDATA[Link Exchange]]></category>
		<category><![CDATA[Link Exchange Tips]]></category>
		<category><![CDATA[Lt]]></category>
		<category><![CDATA[No Tricks]]></category>
		<category><![CDATA[Other Sports]]></category>
		<category><![CDATA[Quick Time]]></category>
		<category><![CDATA[Repetition]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[Scripts]]></category>
		<category><![CDATA[Search Engine Optimization (SEO) Information]]></category>
		<category><![CDATA[search engines]]></category>
		<category><![CDATA[SEO]]></category>
		<category><![CDATA[SEO articles]]></category>
		<category><![CDATA[SEO information]]></category>
		<category><![CDATA[Sessions]]></category>
		<category><![CDATA[Title Tag]]></category>
		<category><![CDATA[Web Directory]]></category>
		<category><![CDATA[what is SEO]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2009/05/29/link-exchange-tips-no-tricks/</guid>
		<description><![CDATA[Use text links, avoid image links. Anyhow, if you have used image links, then always make sure to put your keywords in the alt tags. Put your prime keywords in text links and always insure to put a short descriptions of your website/page in minimum of 20-30 words or more if allowed. In text links [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2009/05/29/link-exchange-tips-no-tricks/feed/</wfw:commentRss>
		<slash:comments>2</slash:comments>
		</item>
		<item>
		<title>Robots.txt or how to get your site properly spidered, crawled,
indexed by bots</title>
		<link>http://www.thaidoweb.com/2009/03/27/robotstxt-or-how-to-get-your-site-properly-spidered-crawledindexed-by-bots/</link>
		<comments>http://www.thaidoweb.com/2009/03/27/robotstxt-or-how-to-get-your-site-properly-spidered-crawledindexed-by-bots/#comments</comments>
		<pubDate>Fri, 27 Mar 2009 09:40:01 +0000</pubDate>
		<dc:creator>admin</dc:creator>
				<category><![CDATA[Web Design]]></category>
		<category><![CDATA[24 Hours]]></category>
		<category><![CDATA[Affiliate Website]]></category>
		<category><![CDATA[Ascii File]]></category>
		<category><![CDATA[Bots]]></category>
		<category><![CDATA[Computer Program]]></category>
		<category><![CDATA[crawlers]]></category>
		<category><![CDATA[Intellectual Property]]></category>
		<category><![CDATA[Lingo]]></category>
		<category><![CDATA[Logs]]></category>
		<category><![CDATA[Mit]]></category>
		<category><![CDATA[Protect Privacy]]></category>
		<category><![CDATA[Protect Your Privacy]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[search engines]]></category>
		<category><![CDATA[Spiders]]></category>
		<category><![CDATA[Web Pages]]></category>
		<category><![CDATA[Web Robot]]></category>
		<category><![CDATA[Www Robot]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2009/03/27/robotstxt-or-how-to-get-your-site-properly-spidered-crawledindexed-by-bots/</guid>
		<description><![CDATA[So you heard about someone stressing the importance of the robots.txt file, or noticed in your website&#8217;s logs that the robots.txt file is causing an error, or somehow it is on the very top of the top visited pages, or, you read some article about the death of the robots.txt file and about how you [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2009/03/27/robotstxt-or-how-to-get-your-site-properly-spidered-crawledindexed-by-bots/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>How to Keep Robots Out of Your Web Site</title>
		<link>http://www.thaidoweb.com/2009/02/02/how-to-keep-robots-out-of-your-web-site/</link>
		<comments>http://www.thaidoweb.com/2009/02/02/how-to-keep-robots-out-of-your-web-site/#comments</comments>
		<pubDate>Sun, 01 Feb 2009 17:40:02 +0000</pubDate>
		<dc:creator>admin</dc:creator>
				<category><![CDATA[Web Design]]></category>
		<category><![CDATA[Accesses]]></category>
		<category><![CDATA[Conventions]]></category>
		<category><![CDATA[crawlers]]></category>
		<category><![CDATA[E Books]]></category>
		<category><![CDATA[Html Document]]></category>
		<category><![CDATA[Meta Content]]></category>
		<category><![CDATA[Meta Name]]></category>
		<category><![CDATA[Meta Tag]]></category>
		<category><![CDATA[Postings]]></category>
		<category><![CDATA[Recursively]]></category>
		<category><![CDATA[Robot]]></category>
		<category><![CDATA[Robots Exclusion Protocol]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[Root Server]]></category>
		<category><![CDATA[search engines]]></category>
		<category><![CDATA[Spiders Web]]></category>
		<category><![CDATA[Web Indexes]]></category>
		<category><![CDATA[Web Page]]></category>
		<category><![CDATA[Web Pages]]></category>
		<category><![CDATA[Web Robots]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2009/02/02/how-to-keep-robots-out-of-your-web-site/</guid>
		<description><![CDATA[THE ROBOTS.TXT FILE You know that search engines have been created to help people find information quickly on the Internet, and the search engines acquire much of their information through robots (also known as spiders or crawlers), that look for web pages for them. The spiders or crawlers robots explore the web looking for and [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2009/02/02/how-to-keep-robots-out-of-your-web-site/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>Harnessing the Power of Robots.txt</title>
		<link>http://www.thaidoweb.com/2009/01/28/harnessing-the-power-of-robotstxt/</link>
		<comments>http://www.thaidoweb.com/2009/01/28/harnessing-the-power-of-robotstxt/#comments</comments>
		<pubDate>Wed, 28 Jan 2009 04:40:02 +0000</pubDate>
		<dc:creator>admin</dc:creator>
				<category><![CDATA[Web Design]]></category>
		<category><![CDATA[Ask Jeeves]]></category>
		<category><![CDATA[Askjeeves]]></category>
		<category><![CDATA[Bot]]></category>
		<category><![CDATA[Further Reading]]></category>
		<category><![CDATA[Linux]]></category>
		<category><![CDATA[Linux Systems]]></category>
		<category><![CDATA[Newsletter]]></category>
		<category><![CDATA[Other Search Engines]]></category>
		<category><![CDATA[Public Html Directory]]></category>
		<category><![CDATA[Reading Resources]]></category>
		<category><![CDATA[Robot]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[Tips And Tricks]]></category>
		<category><![CDATA[User Names]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2009/01/28/harnessing-the-power-of-robotstxt/</guid>
		<description><![CDATA[Once we have a website up and running, we need to make sure that all visiting search engines can access all the pages we want them to look at. Sometimes, we may want search engines to not index certain parts of the site, or even ban other Search Engines from the site all together. This [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2009/01/28/harnessing-the-power-of-robotstxt/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>Tips to Protect Your Downloads or Products</title>
		<link>http://www.thaidoweb.com/2009/01/14/tips-to-protect-your-downloads-or-products/</link>
		<comments>http://www.thaidoweb.com/2009/01/14/tips-to-protect-your-downloads-or-products/#comments</comments>
		<pubDate>Wed, 14 Jan 2009 02:40:03 +0000</pubDate>
		<dc:creator>admin</dc:creator>
				<category><![CDATA[Web Design]]></category>
		<category><![CDATA[Accesses]]></category>
		<category><![CDATA[Cgi Script]]></category>
		<category><![CDATA[Cgi Scripts]]></category>
		<category><![CDATA[Com Downloads]]></category>
		<category><![CDATA[Domain]]></category>
		<category><![CDATA[Ebook Compilers]]></category>
		<category><![CDATA[Ebooks]]></category>
		<category><![CDATA[Folder Name]]></category>
		<category><![CDATA[Folder Test]]></category>
		<category><![CDATA[Folders]]></category>
		<category><![CDATA[Index File]]></category>
		<category><![CDATA[Ip Address]]></category>
		<category><![CDATA[Monster]]></category>
		<category><![CDATA[Password Protection]]></category>
		<category><![CDATA[Permission Denied Error]]></category>
		<category><![CDATA[Protection Features]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[Root Directory]]></category>
		<category><![CDATA[Sentry]]></category>
		<category><![CDATA[Thankyou]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2009/01/14/tips-to-protect-your-downloads-or-products/</guid>
		<description><![CDATA[1. Upload robots.txt file in to your root directory and include the folder name where you set your downloads. More information on how to set robots.txt: http://www.webmasters-central.com/wp/se/robotstxt.shtml 2. Set the permission of the download folder to 711 OR upload an index file to that folder. This makes that folder web inaccessible. For example create a [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2009/01/14/tips-to-protect-your-downloads-or-products/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>The robots.txt file</title>
		<link>http://www.thaidoweb.com/2008/12/16/the-robotstxt-file-2/</link>
		<comments>http://www.thaidoweb.com/2008/12/16/the-robotstxt-file-2/#comments</comments>
		<pubDate>Tue, 16 Dec 2008 11:40:02 +0000</pubDate>
		<dc:creator>admin</dc:creator>
				<category><![CDATA[Web Design]]></category>
		<category><![CDATA[Bot]]></category>
		<category><![CDATA[google]]></category>
		<category><![CDATA[Lycra]]></category>
		<category><![CDATA[Paragraph]]></category>
		<category><![CDATA[Popularity Algorithm]]></category>
		<category><![CDATA[Presence]]></category>
		<category><![CDATA[Regular Expressions]]></category>
		<category><![CDATA[Relative Url]]></category>
		<category><![CDATA[Relative Urls]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[search engines]]></category>
		<category><![CDATA[Server Start]]></category>
		<category><![CDATA[Standard For Robot Exclusion]]></category>
		<category><![CDATA[Web Robots]]></category>
		<category><![CDATA[Webcrawler]]></category>
		<category><![CDATA[Wildcard Patterns]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2008/12/16/the-robotstxt-file-2/</guid>
		<description><![CDATA[Since the beginning of Internet there is a need to index the Web and many robots are built for this purpose. You already know that famous Google bot which is indexing the Web to keep track of urls and build a scheme out of it (link popularity algorithm&#8230;). There are not so many way to [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2008/12/16/the-robotstxt-file-2/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
		<item>
		<title>The robots.txt file</title>
		<link>http://www.thaidoweb.com/2008/11/26/the-robotstxt-file/</link>
		<comments>http://www.thaidoweb.com/2008/11/26/the-robotstxt-file/#comments</comments>
		<pubDate>Tue, 25 Nov 2008 19:40:02 +0000</pubDate>
		<dc:creator>admin</dc:creator>
				<category><![CDATA[Web Design]]></category>
		<category><![CDATA[Bot]]></category>
		<category><![CDATA[google]]></category>
		<category><![CDATA[Lycra]]></category>
		<category><![CDATA[Paragraph]]></category>
		<category><![CDATA[Popularity Algorithm]]></category>
		<category><![CDATA[Presence]]></category>
		<category><![CDATA[Regular Expressions]]></category>
		<category><![CDATA[Relative Url]]></category>
		<category><![CDATA[Relative Urls]]></category>
		<category><![CDATA[Robots Txt]]></category>
		<category><![CDATA[search engines]]></category>
		<category><![CDATA[Server Start]]></category>
		<category><![CDATA[Standard For Robot Exclusion]]></category>
		<category><![CDATA[Web Robots]]></category>
		<category><![CDATA[Webcrawler]]></category>
		<category><![CDATA[Wildcard Patterns]]></category>

		<guid isPermaLink="false">http://www.thaidoweb.com/2008/11/26/the-robotstxt-file/</guid>
		<description><![CDATA[Since the beginning of Internet there is a need to index the Web and many robots are built for this purpose. You already know that famous Google bot which is indexing the Web to keep track of urls and build a scheme out of it (link popularity algorithm&#8230;). There are not so many way to [...]]]></description>
		<wfw:commentRss>http://www.thaidoweb.com/2008/11/26/the-robotstxt-file/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
	</channel>
</rss>

