Use a dedicated taskq for vdev_file

Originally, vdev_file used system_taskq. This would cause a deadlock,
especially on system with few CPUs. The reason is that the prefetcher
threads, which are on system_taskq, will sometimes be blocked waiting
for I/O to finish. If the prefetcher threads consume all the tasks in
system_taskq, the I/O cannot be served and thus results in a deadlock.

We fix this by creating a dedicated vdev_file_taskq for vdev_file I/O.

Signed-off-by: Chunwei Chen <tuxoko@gmail.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #2270

include/sys/vdev_file.h

@@ -39,6 +39,9 @@ typedef struct vdev_file {
 	vnode_t		*vf_vnode;
 } vdev_file_t;
 
+extern void vdev_file_init(void);
+extern void vdev_file_fini(void);
+
 #ifdef	__cplusplus
 }
 #endif

module/zfs/spa_misc.c

@@ -34,6 +34,7 @@
 #include <sys/zap.h>
 #include <sys/zil.h>
 #include <sys/vdev_impl.h>
+#include <sys/vdev_file.h>
 #include <sys/metaslab.h>
 #include <sys/uberblock_impl.h>
 #include <sys/txg.h>
@@ -1660,6 +1661,7 @@ spa_init(int mode)
 	dmu_init();
 	zil_init();
 	vdev_cache_stat_init();
+	vdev_file_init();
 	zfs_prop_init();
 	zpool_prop_init();
 	zpool_feature_init();
@@ -1674,6 +1676,7 @@ spa_fini(void)
 
 	spa_evict_all();
 
+	vdev_file_fini();
 	vdev_cache_stat_fini();
 	zil_fini();
 	dmu_fini();

module/zfs/vdev_file.c

@@ -36,6 +36,8 @@
  * Virtual device vector for files.
  */
 
+static taskq_t *vdev_file_taskq;
+
 static void
 vdev_file_hold(vdev_t *vd)
 {
@@ -184,7 +186,7 @@ vdev_file_io_start(zio_t *zio)
 		return (ZIO_PIPELINE_CONTINUE);
 	}
 
-	VERIFY3U(taskq_dispatch(system_taskq, vdev_file_io_strategy, zio,
+	VERIFY3U(taskq_dispatch(vdev_file_taskq, vdev_file_io_strategy, zio,
 	    TQ_PUSHPAGE), !=, 0);
 
 	return (ZIO_PIPELINE_STOP);
@@ -209,6 +211,21 @@ vdev_ops_t vdev_file_ops = {
 	B_TRUE			/* leaf vdev */
 };
 
+void
+vdev_file_init(void)
+{
+	vdev_file_taskq = taskq_create("vdev_file_taskq", 100, minclsyspri,
+	    max_ncpus, INT_MAX, TASKQ_PREPOPULATE | TASKQ_THREADS_CPU_PCT);
+
+	VERIFY(vdev_file_taskq);
+}
+
+void
+vdev_file_fini(void)
+{
+	taskq_destroy(vdev_file_taskq);
+}
+
 /*
  * From userland we access disks just like files.
  */