创建数据库
create database dc_test
character set gbk;

创建测试表test1
CREATE TABLE `test1` (
  `id` int(11) NOT NULL auto_increment,
  `title` varchar(50) NOT NULL,
  `user_id` int(11) NOT NULL,
  `gmt_create` datetime NOT NULL,
  `gmt_modified` datetime NOT NULL,
  PRIMARY KEY  (`id`),
  KEY `idx_test1_userid` (`user_id`,`gmt_create`)
) ENGINE=InnoDB  DEFAULT CHARSET=gbk

重复写入数据
mysql> insert into test1(title,user_id,gmt_create,gmt_modified) values('this is a test',1001,now(),now());
Query OK, 1 row affected (0.00 sec)

mysql> insert into test1(title,user_id,gmt_create,gmt_modified) select title,user_id,gmt_create,gmt_modified from test1;
Query OK, 786432 rows affected (13.65 sec)
Records: 786432  Duplicates: 0  Warnings: 0

mysql> select count(*) from test1;
+----------+
| count(*) |
+----------+
|  1572864 |
+----------+
1 row in set (0.59 sec)

创建一张只有主键的测试表
CREATE TABLE `test2` (
  `id` int(11) NOT NULL auto_increment,
  `title` varchar(50) NOT NULL,
  `user_id` int(11) NOT NULL,
  `gmt_create` datetime NOT NULL,
  `gmt_modified` datetime NOT NULL,
  PRIMARY KEY  (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=gbk

创建一张没有主键的测试表test3
CREATE TABLE `test3` (
  `id` int(11) NOT NULL,
  `title` varchar(50) NOT NULL,
  `user_id` int(11) NOT NULL,
  `gmt_create` datetime NOT NULL,
  `gmt_modified` datetime NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=gbk

创建innodb_table_monitor表，通知innodb存储引擎将数据输出到/etc/my.cnf中定义的log-error文件里
mysql> create table innodb_table_monitor(a int) engine=innodb;
Query OK, 0 rows affected (0.05 sec)

===========================================
090623  8:15:25 INNODB TABLE MONITOR OUTPUT
===========================================
--------------------------------------
TABLE: name dc_test/innodb_table_monitor, id 0 24, columns 5, indexes 1, appr.rows 0
  COLUMNS: a: DATA_INT len 4 prec 0; DB_ROW_ID: DATA_SYS prtype 256 len 6 prec 0; DB_TRX_ID: DATA_SYS prtype 257 len 6 prec 0; DB_ROLL_PTR: DATA_SYS prtype 258 len 7 prec 0;
  INDEX: name GEN_CLUST_INDEX, id 0 28, fields 0/4, type 1
   root page 50, appr.key vals 0, leaf pages 1, size pages 1
   FIELDS:  DB_ROW_ID DB_TRX_ID DB_ROLL_PTR a
   --
  
--------------------------------------
TABLE: name dc_test/test1, id 0 22, columns 9, indexes 2, appr.rows 1573185
  COLUMNS: id: DATA_INT len 4 prec 0; title: type 12 len 100 prec 0; user_id: DATA_INT len 4 prec 0; gmt_create: DATA_INT len 8 prec 0; gmt_modified: DATA_INT len 8 prec 0; DB_ROW_ID: DATA_SYS prtype 256 len 6 prec 0; DB_TRX_ID: DATA_SYS prtype 257 len 6 prec 0; DB_ROLL_PTR: DATA_SYS prtype 258 len 7 prec 0;
  INDEX: name PRIMARY, id 0 25, fields 1/7, type 3
   root page 57, appr.key vals 1573185, leaf pages 5959, size pages 5991  --注意这个值
   FIELDS:  id DB_TRX_ID DB_ROLL_PTR title user_id gmt_create gmt_modified
  INDEX: name idx_test1_userid, id 0 26, fields 2/3, type 0
   root page 62, appr.key vals 9, leaf pages 2093, size pages 2279
   FIELDS:  user_id gmt_create id  
   --主键与表的数据是存放在一起的,所以primary index(size pages 5991) 比 second index(size pages 2279) 大;从索引字段上看，second index(idx_test1_userid)包含primary indexed columns(id)
--------------------------------------
TABLE: name dc_test/test2, id 0 14, columns 9, indexes 1, appr.rows 0
  COLUMNS: id: DATA_INT len 4 prec 0; title: type 12 len 100 prec 0; user_id: DATA_INT len 4 prec 0; gmt_create: DATA_INT len 8 prec 0; gmt_modified: DATA_INT len 8 prec 0; DB_ROW_ID: DATA_SYS prtype 256 len 6 prec 0; DB_TRX_ID: DATA_SYS prtype 257 len 6 prec 0; DB_ROLL_PTR: DATA_SYS prtype 258 len 7 prec 0;
  INDEX: name PRIMARY, id 0 16, fields 1/7, type 3
   root page 63, appr.key vals 0, leaf pages 1, size pages 1
   FIELDS:  id DB_TRX_ID DB_ROLL_PTR title user_id gmt_create gmt_modified
   --主键与表的数据是存放在一起的
  
--------------------------------------
TABLE: name dc_test/test3, id 0 17, columns 9, indexes 1, appr.rows 0
  COLUMNS: id: DATA_INT len 4 prec 0; title: type 12 len 100 prec 0; user_id: DATA_INT len 4 prec 0; gmt_create: DATA_INT len 8 prec 0; gmt_modified: DATA_INT len 8 prec 0; DB_ROW_ID: DATA_SYS prtype 256 len 6 prec 0; DB_TRX_ID: DATA_SYS prtype 257 len 6 prec 0; DB_ROLL_PTR: DATA_SYS prtype 258 len 7 prec 0;
  INDEX: name GEN_CLUST_INDEX, id 0 19, fields 0/8, type 1
   root page 213, appr.key vals 0, leaf pages 1, size pages 1
   FIELDS:  DB_ROW_ID DB_TRX_ID DB_ROLL_PTR id title user_id gmt_create gmt_modified
   --在创建表时，没有定义主键，innodb会在内部创建一个主键索引
----------------------------------
END OF INNODB TABLE MONITOR OUTPUT
==================================

关于上面一些特殊字段中的含义
DB_TRX_ID - this field is managed by InnoDB internally and contains a ID of transaction which changed a record last time
DB_ROLL_PTR - one more internal InnoDB field (TODO: find out what is it used for).
DB_ROW_ID - this internally used field should be the first field in tables without primary keys (it is an auto-increment field used by InnoDB to identify rows in such tables)

test1表大小为5991 pages,与show table status like 'test1'中的Data_length相比，数据计算证明完全相同
mysql> show table status like 'test1'\G
*************************** 1. row ***************************
           Name: test1
         Engine: InnoDB
        Version: 10
     Row_format: Compact
           Rows: 1573185  --这个是抽样值，不是精确值
 Avg_row_length: 62
    Data_length: 98156544
Max_data_length: 0
   Index_length: 37339136
      Data_free: 0
 Auto_increment: 1572865
    Create_time: 2009-06-22 12:37:17
    Update_time: NULL
     Check_time: NULL
      Collation: gbk_chinese_ci
       Checksum: NULL
 Create_options:
        Comment: InnoDB free: 871424 kB
1 row in set (0.00 sec)

mysql> select 5991*16*1024;
+--------------+
| 5991*16*1024 |
+--------------+
|     98156544 |
+--------------+
1 row in set (0.00 sec)

参考文档:
http://code.google.com/p/innodb-tools/wiki/InnodbRecoveryHowto

老婆的照片pic1:

pic2:

pic3:

在westlake做早操的我,哈哈

如果有一个key,如何获取value值?包括如下的几个步骤：

1.如果key为字符型，计算key的md5,取其前几个字节，map到对应的数据结点.如果key为数字类型，可以通过取模的方式来hash化。这也是我们通常在做数据库水平拆分sharding时，常采用的做法。key-value分布式存储系统，通常用来存储文件，如果保存的文件个数比较多，hash的粒度会比较大，以保证单数据结点的数据比较小。为了数据安全，一份数据在写入时，会向多个数据结点复制，采用这种办法，也可提高数据的并发读取性能。

2.查找index文件,应尽量保证索引文件cache到内存中

3.查找data文件

参考文章：

http://project-voldemort.com/blog/

个人理解，淘宝旺店的产品目标用户应该是定位在高端卖家。这个差异化产品的推出，大家也可感受到淘宝战略方向的变化。淘宝越来越平台化，为个体，企业级卖家提供各种电子商务接入服务。

    大家可以上优衣库中国网站，体验一下。以上观点，只代表我本人，不代表官方任何立场。

     以下是腾讯网上公布的一些优秀作文，读起来别有一番风味：

     合肥考生的《弯道超越》

     北京考生的《我有一双隐形的翅膀》

     希望看到越来越多这样的文章...

    BLOCKS
----------
      1280

1 row selected.

09:52:55 danchen@ test_oracle>select /*+ rowid(u)*/  count(*) from test_dc u where rowid>=CHARTOROWID('AAADQdAA6AAADcJAAA') and rowid<=CHARTOROWID('AAADQdAA6AAADwICcQ');

  COUNT(*)
----------

     22852

09:54:55 danchen@ test_oracle>select 1280*8*1024/22852 bytes from dual;

     BYTES
----------
458.855242

--在mysql作对比测试
root@test_mysql 09:47:47>select count(*) from test_dc_181;
+----------+
| count(*) |
+----------+
|   128928 |
+----------+
1 row in set (0.08 sec)

root@test_mysql 09:45:57>show table status like 'test_dc_181'\G;
*************************** 1. row ***************************
           Name: test_dc_181
         Engine: InnoDB
        Version: 10
     Row_format: Compact
           Rows: 85668   --统计信息不准确
 Avg_row_length: 1122
    Data_length: 96141312
Max_data_length: 0
   Index_length: 0
      Data_free: 0
 Auto_increment: NULL
    Create_time: 2009-05-27 11:23:55
    Update_time: NULL
     Check_time: NULL
      Collation: gbk_chinese_ci
       Checksum: NULL
 Create_options:
        Comment: InnoDB free: 292984832 kB
1 row in set (0.00 sec)

ERROR:
No query specified

--重新分析一下,发现执行分析操作时间非常快，数据也比较准确，估计mysql的分析的算法原理为基于抽样的统计，比如说一个page里有多少记录，总的page数是多少,两者相乘即可得到行数。
root@test_mysql 09:46:01>analyze table test_dc_181;
+---------------------+---------+----------+----------+
| Table               | Op      | Msg_type | Msg_text |
+---------------------+---------+----------+----------+
| test_mysql.test_dc_181 | analyze | status   | OK       |
+---------------------+---------+----------+----------+
1 row in set (0.00 sec)

root@test_mysql 09:46:46>show table status like 'test_dc_181'\G;
*************************** 1. row ***************************
           Name: test_dc_181
         Engine: InnoDB
        Version: 10
     Row_format: Compact
           Rows: 115074
 Avg_row_length: 835
    Data_length: 96141312
Max_data_length: 0
   Index_length: 0
      Data_free: 0
 Auto_increment: NULL
    Create_time: 2009-05-27 11:23:55
    Update_time: NULL
     Check_time: NULL
      Collation: gbk_chinese_ci
       Checksum: NULL
 Create_options:
        Comment: InnoDB free: 292984832 kB
1 row in set (0.00 sec)

ERROR:
No query specified

--在mysql innodb的表中平均单行长度为835字节

root@test_mysql 09:46:50>select 96141312/115074;
+-----------------+
| 96141312/115074 |
+-----------------+
|        835.4738 |
+-----------------+
1 row in set (0.00 sec)

--在此测试用例下，同一行记录，在oracle与mysql存储空间对比为1.8:1,在其它大多数情况下，oracle的数据迁移到mysql数据库，所用的空间都要大许多，具体大多少，各位需要经过测试，以使容量估计更加准确。
root@test_mysql 09:48:14>select 835.4738/458.855242 ;
+---------------------+
| 835.4738/458.855242 |
+---------------------+
|          1.82077859 |
+---------------------+
1 row in set (0.00 sec)

   The Photos application is one of Facebook’s most popular features. Up to date, users have uploaded over 15 billion photos which makes Facebook the biggest photo sharing website. For each uploaded photo, Facebook generates and stores four images of different sizes, which translates to a total of 60 billion images and 1.5PB of storage. The current growth rate is 220 million new photos per week, which translates to 25TB of additional storage consumed weekly. At the peak there are 550,000 images served per second. These numbers pose a significant challenge for the Facebook photo storage infrastructure.

NFS photo infrastructure

The old photo infrastructure consisted of several tiers:

• Upload tier receives users’ photo uploads, scales the original images and saves them on the NFS storage tier.
• Photo serving tier receives HTTP requests for photo images and serves them from the NFS storage tier.
• NFS storage tier built on top of commercial storage appliances.

Since each image is stored in its own file, there is an enormous amount of metadata generated on the storage tier due to the namespace directories and file inodes. The amount of metadata far exceeds the caching abilities of the NFS storage tier, resulting in multiple I/O operations per photo upload or read request. The whole photo serving infrastructure is bottlenecked on the high metadata overhead of the NFS storage tier, which is one of the reasons why Facebook relies heavily on CDNs to serve photos. Two additional optimizations were deployed in order to mitigate this problem to some degree:

• Cachr: a caching server tier caching smaller Facebook "profile" images.
• NFS file handle cache - deployed on the photo serving tier eliminates some of the NFS storage tier metadata overhead

Haystack Photo Infrastructure

The new photo infrastructure merges the photo serving tier and storage tier into one physical tier. It implements a HTTP based photo server which stores photos in a generic object store called Haystack. The main requirement for the new tier was to eliminate any unnecessary metadata overhead for photo read operations, so that each read I/O operation was only reading actual photo data (instead of filesystem metadata). Haystack can be broken down into these functional layers -

• HTTP server
• Photo Store
• Haystack Object Store
• Filesystem
• Storage

In the following sections we look closely at each of the functional layers from the bottom up.

Storage

Haystack is deployed on top of commodity storage blades. The typical hardware configuration of a 2U storage blade is –

• 2 x quad-core CPUs
• 16GB – 32GB memory
• hardware raid controller with 256MB – 512MB of NVRAM cache
• 12+ 1TB SATA drives

Each storage blade provides around 10TB of usable space, configured as a RAID-6 partition managed by the hardware RAID controller. RAID-6 provides adequate redundancy and excellent read performance while keeping the storage cost down. The poor write performance is partially mitigated by the RAID controller NVRAM write-back cache. Since the reads are mostly random, the NVRAM cache is fully reserved for writes. The disk caches are disabled in order to guarantee data consistency in the event of a crash or a power loss.

Filesystem

Haystack object stores are implemented on top of files stored in a single filesystem created on top of the 10TB volume.

Photo read requests result in read() system calls at known offsets in these files, but in order to execute the reads, the filesystem must first locate the data on the actual physical volume. Each file in the filesystem is represented by a structure called an inode which contains a block map that maps the logical file offset to the physical block offset on the physical volume. For large files, the block map can be quite large depending on the type of the filesystem in use.

Block based filesystems maintain mappings for each logical block, and for large files, this information will not typically fit into the cached inode and is stored in indirect address blocks instead, which must be traversed in order to read the data for a file. There can be several layers of indirection, so a single read could result in several I/Os depending on whether or not the indirect address blocks are cached.

Extent based filesystems maintain mappings only for contiguous ranges of blocks (extents). A block map for a contiguous large file could consist of only one extent which would fit in the inode itself. However, if the file is severely fragmented and its blocks are not contiguous on the underlying volume, its block map can grow large as well. With extent based filesystems, fragmentation can be mitigated by aggressively allocating a large chunk of space whenever growing the physical file.

Currently, the filesystem of choice is XFS, an extent based filesystem providing efficient file preallocation.

Haystack Object Store

Haystack is a simple log structured (append-only) object store containing needles representing the stored objects. A Haystack consists of two files – the actual haystack store file containing the needles, plus an index file. The following figure shows the layout of the haystack store file:
The first 8KB of the haystack store is occupied by the superblock. Immediately following the superblock are needles, with each needle consisting of a header, the data, and a footer:
A needle is uniquely identified by its <Offset, Key, Alternate Key, Cookie> tuple, where the offset is the needle offset in the haystack store. Haystack doesn’t put any restriction on the values of the keys, and there can be needles with duplicate keys. Following figure shows the layout of the index file -
There is a corresponding index record for each needle in the haystack store file, and the order of the needle index records must match the order of the associated needles in the haystack store file. The index file provides the minimal metadata required to locate a particular needle in the haystack store file. Loading and organizing index records into a data structure for efficient lookup is the responsibility of the Haystack application (Photo Store in our case). The index file is not critical, as it can be rebuilt from the haystack store file if required. The main purpose of the index is to allow quick loading of the needle metadata into memory without traversing the larger Haystack store file, since the index is usually less than 1% the size of the store file.

Haystack Write Operation

A Haystack write operation synchronously appends new needles to the haystack store file. After the needles are committed to the larger Haystack store file, the corresponding index records are then written to the index file. Since the index file is not critical, the index records are written asynchronously for faster performance.

The index file is also periodically flushed to the underlying storage to limit the extent of the recovery operations caused by hardware failures. In the case of a crash or a sudden power loss, the haystack recovery process discards any partial needles in the store and truncates the haystack store file to the last valid needle. Next, it writes missing index records for any trailing orphan needles at the end of the haystack store file.

Haystack doesn’t allow overwrite of an existing needle offset, so if a needle’s data needs to be modified, a new version of it must be written using the same <Key, Alternate Key, Cookie> tuple. Applications can then assume that among the needles with duplicate keys, the one with the largest offset is the most recent one.

Haystack Read Operation

The parameters passed to the haystack read operation include the needle offset, key, alternate key, cookie and the data size. Haystack then adds the header and footer lengths to the data size and reads the whole needle from the file. The read operation succeeds only if the key, alternate key and cookie match the ones passed as arguments, if the data passes checksum validation, and if the needle has not been previously deleted (see below).

Haystack Delete Operation

The delete operation is simple – it marks the needle in the haystack store as deleted by setting a “deleted” bit in the flags field of the needle. However, the associated index record is not modified in any way so an application could end up referencing a deleted needle. A read operation for such a needle will see the “deleted” flag and fail the operation with an appropriate error. The space of a deleted needle is not reclaimed in any way. The only way to reclaim space from deleted needles is to compact the haystack (see below).

Photo Store Server.

Photo Store Server is responsible for accepting HTTP requests and translating them to the corresponding Haystack store operations. In order to minimize the number of I/Os required to retrieve photos, the server keeps an in-memory index of all photo offsets in the haystack store file. At startup, the server reads the haystack index file and populates the in-memory index. With hundreds of millions of photos per node (and the number will only grow with larger capacity drives), we need to make sure that the index will fit into the available memory. This is achieved by keeping a minimal amount of metadata in memory, just the information required to locate the images.

When a user uploads a photo, it is assigned a unique 64-bit id. The photo is then scaled down to 4 different sizes. Each scaled image has the same random cookie and 64-bit key, and the logical image size (large, medium, small, thumbnail) is stored in the alternate key. The upload server then calls the photo store server to store all four images in the Haystack.

The in-memory index keeps the following information for each photo:
 Haystack uses the open source Google sparse hash data structure to keep the in-memory index small, since it only has 2 bits of overhead per entry.

Photo Store Write/Modify Operation

A write operation writes photos to the haystack and updates the in-memory index with the new entries. If the index already contains records with the same keys then this is a modification of existing photos and only the index records offsets are modified to reflect the location of the new images in the haystack store file. Photo store always assumes that if there are duplicate images (images with the same key) it is the one stored at a larger offset which is valid.

Photo Store Read Operation

The parameters passed to a read operation include haystack id and a photo key, size and cookie. The server performs a lookup in the in-memory index based on the photo key and retrieves the offset of the needle containing the requested image. If found it calls the haystack read operation to get the image. As noted above haystack delete operation doesn’t update the haystack index file record. Therefore a freshly populated in-memory index can contain stale entries for the previously deleted photos. Read of a previously deleted photo will fail and the in-memory index is updated to reflect that by setting the offset of the particular image to zero.

Photo Store Delete Operation

After calling the haystack delete operation the in-memory index is updated by setting the image offset to zero signifying that the particular image has been deleted.

Compaction

Compaction is an online operation which reclaims the space used by the deleted and duplicate needles (needles with the same key). It creates a new haystack by copying needles while skipping any duplicate or deleted entries. Once done it swaps the files and in-memory structures.

HTTP Server

The HTTP framework we use is the simple evhttp server provided with the open source libevent library. We use multiple threads, with each thread being able to serve a single HTTP request at a time. Because our workload is mostly I/O bound, the performance of the HTTP server is not critical.

Summary

Haystack presents a generic HTTP-based object store containing needles that map to stored opaque objects. Storing photos as needles in the haystack eliminates the metadata overhead by aggregating hundreds of thousands of images in a single haystack store file. This keeps the metadata overhead very small and allows us to store each needle’s location in the store file in an in-memory index. This allows retrieval of an image’s data in a minimal number of I/O operations, eliminating all unnecessary metadata overhead.

Peter Vajgel, Doug Beaver and Jason Sobel are infrastructure engineers at Facebook.

原文网址：http://www.facebook.com/note.php?note_id=76191543919&ref=mf

我试验了几把，搜索结果taobao上的商品基本上都是排在第一位的。但是如果越来越多的人都是通过google的购物搜索，直接转到taobao的item页面,后果是相当严重的。这种严重性在于不仅淘宝失去了自身的搜索流量，而且还被别人控制了咽喉。除了购物搜索之外，在google的产品线中还包括博客搜索，图片搜索，音乐搜索，视频搜索，学术搜索等等，这些垂直类的搜索，提高了google在每一个方面的搜索精度，更加强化了google在每个垂直领域内的竞争实力。每个搜索产品的推出，依靠其强大的搜索技术实力，以及出色的产品，不断危协着每一个领域内的其它公司,也更加强化其在互联网搜索领域内的霸主地位。

      一个优秀的技术驱动创新型企业，总是在不断的技术与产品创新中，给我们带来不一样的震撼。

1.Scalability Improvements

MySQL 5.4 modifications improve performance on SMP systems to increase scalability on multi-core systems. The changes affect InnoDB locking, memory management, and thread concurrency. Much of this work is based on a patch developed by Google, about which more information can be found here:

http://code.google.com/p/google-mysql-tools/wiki/SmpPerformance

2.InnoDB I/O Subsystem Changes

MySQL 5.4 changes to the InnoDB I/O subsystem enable more effective use of available I/O capacity. 
The changes also provide more control over configuration of the I/O subsystem. Much of this work 
is based on patches developed by Google, about which more information can be found here: 
http://code.google.com/p/google-mysql-tools/wiki/InnodbAsyncIo
http://code.google.com/p/google-mysql-tools/wiki/InnodbIoTuning
The patch makes the number of background I/O threads configurable via system variables: 
    innodb_read_io_threads controls the number of threads to use for read prefetch requests. 
    innodb_write_io_threads controls the number of threads to use for writing dirty pages from the buffer cache to disk. 
The default for both variables is 8.
随着磁盘IOPS能力越来越高，比如说SSD的出现，在5.4的mysql版本中，还增加了一个参数来表示系统可以提供的IOPS为多少，这个参数为：
innodb_io_capacity 
还有一个新引入的参数innodb_extra_dirty_writes,这个参数是跟innodb_max_dirty_pages_pct一起配合使用,新引入的这个参数是对脏页刷新
控制的一个重要补充。当数据库脏页比率低于innodb_max_dirty_pages_pct时，如果数据库处于相对空闲状态，数据库也会把脏页刷到磁盘上。
 
3.提高了在Solaris平台使用大页的支持
4.show innodb status输出有一些补充，主要是后台线程，SEMAPHORES等等。
 
当前5.4处于beta版，想尝鲜的，可以到如下的地址去下载：
http://dev.mysql.com/downloads/mysql/5.4.html