#!/usr/bin/env -iS perl
#
# $Id$
#
# Copyright (c) 2008 Ben Rockwood <benr@cuddletech.com>,
# Copyright (c) 2010-2011 Jason J. Hellenthal <jhell@DataIX.net>,
# Copyright (c) 2010-2019 Martin Matuska <mm@FreeBSD.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# zfs-stats is a fork of arc_summary.pl maintained by Jason J. Hellenthal
# http://code.google.com/p/jhell/
#
# If you are having troubles when using this script from cron(8) please try
# adjusting your PATH before reporting problems.
#
# /usr/bin & /sbin
#
# Binaries used are:
#
# kldstat(8), sysctl(8) & vmstat(8)

use strict;
use warnings;
use Getopt::Long;
use Scalar::Util qw(looks_like_number);

Getopt::Long::Configure ("bundling");

my $version = '1.3.2';

my $usetunable = 1;			# Change to 0 to disable sysctl MIB spill.
my $show_sysctl_descriptions = 0;	# Change to 1 (or use the -d flag) to show sysctl descriptions.
my $alternate_sysctl_layout = 0;	# Change to 1 (or use the -t flag) for alternate output.
my $raw_data;				# Raw data

my $Kstat;

my %cmds = (
	sysctl_q   => '/sbin/sysctl -q',
	uname_v  => 'uname -v',
	uptime   => 'uptime',
	kldstat  => 'kldstat',
	vmstat_m => '/usr/bin/vmstat -m',
);

my %ssh;

sub run{
	my $cmd = shift;
	if( exists $cmds{$cmd} ){
		$cmd = $cmds{$cmd};
	}

	my @ssh;
	if( $ssh{hostname} ){
		push @ssh, 'ssh', $ssh{hostname};
		push @ssh, '-l', $ssh{user}    if $ssh{user};
		push @ssh, '-i', $ssh{id_file} if $ssh{id_file};
		push @ssh, '-p', $ssh{port}    if $ssh{port};
	}

	return `@ssh $cmd @_`;
}

sub _usage {
	print STDERR << "EOF";
Usage: $0 [-ABDHLMSabdhus]

-a	: all statistics (-IMAELZDs)

-I	: general system information
-M	: system memory information
-s	: sysctl ZFS tunables
-d	: descriptions for ZFS tunables
-t	: alternate sysctl layout (arc_summary.pl)

-A	: ARC summary
-E	: ARC efficiency
-D	: VDEV cache statistics
-L	: L2 ARC statistics
-O	: dataset operations statistics
-Z	: DMU (zfetch) statistics

-R	: display raw numbers and bytes
-V	: display program version and exit
-h	: display this (help) message

example: $0 -a

Additionally this command can give you information
on another computer through ssh.

--hostname  : address or hostname of the remote computer
--port      : port of the ssh server on the remote computer
--user      : user name on the remote computer
--id_file   : path to the identity file to use

example: $0 -a --hostname server.local --user somebody
EOF
	exit;
}

sub _version {
	printf("zfs-stats version %s\n", $version);
	exit 0
}

sub div1 {
	print "\n";
	print '-' x 72;
	print "\n";
}

sub div2 { div1; print "\n"; }

my @binary_prefix = (
	['YiB', 2 ** 80],
	['ZiB', 2 ** 70],
	['EiB', 2 ** 60],
	['PiB', 2 ** 50],
	['TiB', 2 ** 40],
	['GiB', 2 ** 30],
	['MiB', 2 ** 20],
	['KiB', 2 ** 10],
);
sub fBytes {
	my ( $Bytes, $Decimal ) = @_;
	if ($raw_data) { return $Bytes };

	defined($Bytes) or $Bytes = 0;
	defined($Decimal) or $Decimal = 2;

	for( @binary_prefix ){
		my($prefix,$cmp) = @$_;

		next if $Bytes < $cmp;
		return sprintf("%0.${Decimal}f\t%s", $Bytes / $cmp, $prefix );
	}

	return "$Bytes\tBytes";
}

my @hits_compare = (
	['S', 10**24],
	['s', 10**21],
	['Q', 10**18],
	['q', 10**15],
	['t', 10**12],
	['b', 10**9 ],
	['m', 10**6 ],
	['k', 10**3 ],
);
sub fHits {
	my ( $Hits, $Decimal ) = @_;
	if ($raw_data) { return $_[0] }

	defined($Hits) or $Hits = 0;
	defined($Decimal) or $Decimal = 2;

	for( @hits_compare ){
		my($prefix,$cmp) = @$_;

		next if $Hits < $cmp;
		return sprintf("%0.${Decimal}f\t%s", $Hits / $cmp, $prefix );
	}

	return $Hits;
}

sub fPerc {
	my $lVal = $_[0] || 0;
	defined($lVal) or $lVal = 0;

	my $rVal = $_[1] || 0;
	defined($rVal) or $rVal = 0;

	my $Decimal = $_[2] || 2;
	defined($Decimal) or $Decimal = 2;

	if ($rVal > 0) {
		return sprintf('%0.' . $Decimal . 'f', 100*($lVal / $rVal)) . "%";
	} else { return sprintf('%0.' . $Decimal . 'f', 100) . "%"; }
}

sub fRatio {
	my $lVal = $_[0] // 0;
	my $rVal = $_[1] // 1;
	my $Decimal = $_[2] // 2;

	return sprintf('%0.' . $Decimal . 'f', ($lVal / $rVal));
}

my @Kstats = qw(
	hw.machine
	hw.machine_arch
	hw.pagesize
	hw.physmem
	kern.maxusers
	kern.osreldate
	vm.kmem_map_free
	vm.kmem_map_size
	vm.kmem_size
	vm.kmem_size_max
	vm.kmem_size_min
	vm.kmem_size_scale
	vm.stats
	kstat.zfs.misc.abdstats
	kstat.zfs.misc.arcstats
	kstat.zfs.misc.dbufstats
	kstat.zfs.misc.dmu_tx
	kstat.zfs.misc.dnodestats
	kstat.zfs.misc.fletcher_4_bench
	kstat.zfs.misc.fm
	kstat.zfs.misc.vdev_cache_stats
	kstat.zfs.misc.vdev_mirror_stats
	kstat.zfs.misc.vdev_raidz_bench
	kstat.zfs.misc.xuio_stats
	kstat.zfs.misc.zfetchstats
	kstat.zfs.misc.zil
	kstat.zfs.misc.zstd
	vfs.zfs
);

my $IsOpenZFS = 1;

sub _start {
	my $daydate = localtime; chomp $daydate;
	$IsOpenZFS = defined($Kstat->{"vfs.zfs.vdev.cache_size"});
	div1;
	printf("ZFS Subsystem Report\t\t\t\t%s", $daydate);
	div2;
}

sub _system_info {
	my $unamev = run('uname_v');
	$unamev =~ s/@.*//;
	chomp $unamev;
	$unamev =~ s/\s+/ /g;
	my $sysuptime = run('uptime');
	print "System Information:\n";
	print "\n";
	printf("\tKernel Version:\t\t\t\t%d (osreldate)\n", $Kstat->{"kern.osreldate"});
	printf("\tHardware Platform:\t\t\t%s\n", $Kstat->{"hw.machine"});
	printf("\tProcessor Architecture:\t\t\t%s\n", $Kstat->{"hw.machine_arch"});
	print "\n";
	printf("\tZFS Storage pool Version:\t\t%d\n", $Kstat->{"vfs.zfs.version.spa"});
	printf("\tZFS Filesystem Version:\t\t\t%d\n", $Kstat->{"vfs.zfs.version.zpl"});
	print "\n";
	printf("%s", $unamev);
	printf("%s", $sysuptime);
}

sub _system_memory {
	sub mem_rounded {
		my ($mem_size) = @_;
		my $chip_size = 1;
		my $chip_guess = ($mem_size / 8) - 1;
		while ($chip_guess != 0) {
			$chip_guess >>= 1;
			$chip_size <<= 1;
		}
		my $mem_round = (int($mem_size / $chip_size) + 1) * $chip_size;
		return $mem_round;
	}

	my $pagesize = $Kstat->{"hw.pagesize"};
	my $mem_hw = &mem_rounded($Kstat->{"hw.physmem"});
	my $mem_phys = $Kstat->{"hw.physmem"};
	my $mem_all = $Kstat->{"vm.stats.vm.v_page_count"} * $pagesize;
	my $mem_wire = $Kstat->{"vm.stats.vm.v_wire_count"} * $pagesize;
	my $mem_active = $Kstat->{"vm.stats.vm.v_active_count"} * $pagesize;
	my $mem_inactive = $Kstat->{"vm.stats.vm.v_inactive_count"} * $pagesize;
	my $mem_cache = $Kstat->{"vm.stats.vm.v_cache_count"} * $pagesize;
	my $mem_free = $Kstat->{"vm.stats.vm.v_free_count"} * $pagesize;

	my $mem_gap_vm = $mem_all - ($mem_wire + $mem_active + $mem_inactive + $mem_cache + $mem_free);

	my $mem_total = $mem_hw;
	my $mem_avail = $mem_inactive + $mem_cache + $mem_free;
	my $mem_used = $mem_total - $mem_avail;

	print "System Memory:\n";
	print "\n";
	printf("\t%s\t%s Active,\t", fPerc($mem_active, $mem_all), fBytes($mem_active));
	printf("%s\t%s Inact\n", fPerc($mem_inactive, $mem_all), fBytes($mem_inactive));
	printf("\t%s\t%s Wired,\t", fPerc($mem_wire, $mem_all), fBytes($mem_wire));
	printf("%s\t%s Cache\n", fPerc($mem_cache, $mem_all), fBytes($mem_cache));
	printf("\t%s\t%s Free,\t", fPerc($mem_free, $mem_all), fBytes($mem_free));
	printf("%s\t%s Gap\n", fPerc($mem_gap_vm, $mem_all), fBytes($mem_gap_vm));
	print "\n";
	printf("\tReal Installed:\t\t\t\t%s\n", fBytes($mem_hw));
	printf("\tReal Available:\t\t\t%s\t%s\n", fPerc($mem_phys, $mem_hw), fBytes($mem_phys));
	printf("\tReal Managed:\t\t\t%s\t%s\n", fPerc($mem_all, $mem_phys), fBytes($mem_all));
	print "\n";
	printf("\tLogical Total:\t\t\t\t%s\n", fBytes($mem_total));
	printf("\tLogical Used:\t\t\t%s\t%s\n", fPerc($mem_used, $mem_total), fBytes($mem_used));
	printf("\tLogical Free:\t\t\t%s\t%s\n", fPerc($mem_avail, $mem_total), fBytes($mem_avail));
	print "\n";

	my @ktext = map { /0x/ && hex( (split)[3] ) } run('kldstat'); # retrieve the size column and convert from hex
	my $ktext;
	$ktext += $_ for @ktext; # add the elements of size up

	my @kdata = map { /([0-9]+)K/ } run('vmstat_m'); # retrieve the MemUse column
	my $kdata;
	$kdata += $_ for @kdata; # add the elements of MemUse up
	$kdata *= 1024; # convert from KiB to Bytes

	my $kmem = ($ktext + $kdata);
	my $kmem_map_size = $Kstat->{"vm.kmem_map_size"};
	my $kmem_map_free = $Kstat->{"vm.kmem_map_free"};
	my $kmem_map_total = ($kmem_map_size + $kmem_map_free);

	printf("Kernel Memory:\t\t\t\t\t%s\n", fBytes($kmem));
	printf("\tData:\t\t\t\t%s\t%s\n", fPerc($kdata, $kmem), fBytes($kdata));
	printf("\tText:\t\t\t\t%s\t%s\n\n", fPerc($ktext, $kmem), fBytes($ktext));

	printf("Kernel Memory Map:\t\t\t\t%s\n", fBytes($kmem_map_total));
	printf("\tSize:\t\t\t\t%s\t%s\n", fPerc($kmem_map_size, $kmem_map_total), fBytes($kmem_map_size));
	printf("\tFree:\t\t\t\t%s\t%s\n", fPerc($kmem_map_free, $kmem_map_total), fBytes($kmem_map_free));
}


sub _arc_summary {
	my $memory_throttle_count = $Kstat->{"kstat.zfs.misc.arcstats.memory_throttle_count"};

	print "ARC Summary: ";
	if ($memory_throttle_count > 0) {
		print "(THROTTLED)\n";
	} else { print "(HEALTHY)\n"; }
	printf("\tMemory Throttle Count:\t\t\t%s\n", fHits($memory_throttle_count));
	print "\n";

	### ARC Misc. ###
	my $deleted = $Kstat->{"kstat.zfs.misc.arcstats.deleted"};
	my $evict_skip = $Kstat->{"kstat.zfs.misc.arcstats.evict_skip"};
	my $mutex_miss = $Kstat->{"kstat.zfs.misc.arcstats.mutex_miss"};

	print "ARC Misc:\n";
	printf("\tDeleted:\t\t\t\t%s\n", fHits($deleted));
	printf("\tMutex Misses:\t\t\t\t%s\n", fHits($mutex_miss));
	printf("\tEvict Skips:\t\t\t\t%s\n", fHits($evict_skip));
	print "\n";

	### ARC Sizing ###
	my $arc_size = $Kstat->{"kstat.zfs.misc.arcstats.size"};
	my $arc_compressed_size = $Kstat->{"kstat.zfs.misc.arcstats.compressed_size"};
	my $arc_uncompressed_size = $Kstat->{"kstat.zfs.misc.arcstats.uncompressed_size"};
	my $mru_size = $Kstat->{"kstat.zfs.misc.arcstats.p"} || $Kstat->{"kstat.zfs.misc.arcstats.mru_size"};
	my $target_max_size = $Kstat->{"kstat.zfs.misc.arcstats.c_max"};
	my $target_min_size = $Kstat->{"kstat.zfs.misc.arcstats.c_min"};
	my $target_size = $Kstat->{"kstat.zfs.misc.arcstats.c"};

	my $target_size_ratio = ($target_max_size / $target_min_size);

	printf("ARC Size:\t\t\t\t%s\t%s\n",
		fPerc($arc_size, $target_max_size),  fBytes($arc_size));
	printf("\tTarget Size: (Adaptive)\t\t%s\t%s\n",
		fPerc($target_size, $target_max_size), fBytes($target_size));
	printf("\tMin Size (Hard Limit):\t\t%s\t%s\n",
		fPerc($target_min_size, $target_max_size), fBytes($target_min_size));
	printf("\tMax Size (High Water):\t\t%d:1\t%s\n",
		$target_size_ratio, fBytes($target_max_size));
	printf("\tCompressed Data Size:\t\t\t%s\n",
		fBytes($arc_compressed_size));
	printf("\tDecompressed Data Size:\t\t\t%s\n",
		fBytes($arc_uncompressed_size));
	printf("\tCompression Factor:\t\t\t%s\n",
		fRatio($arc_uncompressed_size, $arc_compressed_size));


	print "\nARC Size Breakdown:\n";
	if ($arc_size > $target_size) {
		my $mfu_size = ($arc_size - $mru_size);
		printf("\tRecently Used Cache Size:\t%s\t%s\n",
			fPerc($mru_size, $arc_size), fBytes($mru_size));
		printf("\tFrequently Used Cache Size:\t%s\t%s\n",
			fPerc($mfu_size, $arc_size), fBytes($mfu_size));
	}

	if ($arc_size < $target_size) {
		my $mfu_size = ($target_size - $mru_size);
		printf("\tRecently Used Cache Size:\t%s\t%s\n",
			fPerc($mru_size, $target_size), fBytes($mru_size));
		printf("\tFrequently Used Cache Size:\t%s\t%s\n",
			fPerc($mfu_size, $target_size), fBytes($mfu_size));
	}
	print "\n";

	### ARC Hash Breakdown ###
	my $hash_chain_max = $Kstat->{"kstat.zfs.misc.arcstats.hash_chain_max"};
	my $hash_chains = $Kstat->{"kstat.zfs.misc.arcstats.hash_chains"};
	my $hash_collisions = $Kstat->{"kstat.zfs.misc.arcstats.hash_collisions"};
	my $hash_elements = $Kstat->{"kstat.zfs.misc.arcstats.hash_elements"};
	my $hash_elements_max = $Kstat->{"kstat.zfs.misc.arcstats.hash_elements_max"};

	print "ARC Hash Breakdown:\n";
	printf("\tElements Max:\t\t\t\t%s\n", fHits($hash_elements_max));
	printf("\tElements Current:\t\t%s\t%s\n",
		fPerc($hash_elements, $hash_elements_max), fHits($hash_elements));
	printf("\tCollisions:\t\t\t\t%s\n", fHits($hash_collisions));
	printf("\tChain Max:\t\t\t\t%s\n", fHits($hash_chain_max));
	printf("\tChains:\t\t\t\t\t%s\n", fHits($hash_chains));
}

sub _arc_efficiency {
	my $arc_hits = $Kstat->{"kstat.zfs.misc.arcstats.hits"};
	my $arc_misses = $Kstat->{"kstat.zfs.misc.arcstats.misses"};
	my $demand_data_hits = $Kstat->{"kstat.zfs.misc.arcstats.demand_data_hits"};
	my $demand_data_misses = $Kstat->{"kstat.zfs.misc.arcstats.demand_data_misses"};
	my $demand_metadata_hits = $Kstat->{"kstat.zfs.misc.arcstats.demand_metadata_hits"};
	my $demand_metadata_misses = $Kstat->{"kstat.zfs.misc.arcstats.demand_metadata_misses"};
	my $mfu_ghost_hits = $Kstat->{"kstat.zfs.misc.arcstats.mfu_ghost_hits"};
	my $mfu_hits = $Kstat->{"kstat.zfs.misc.arcstats.mfu_hits"};
	my $mru_ghost_hits = $Kstat->{"kstat.zfs.misc.arcstats.mru_ghost_hits"};
	my $mru_hits = $Kstat->{"kstat.zfs.misc.arcstats.mru_hits"};
	my $prefetch_data_hits = $Kstat->{"kstat.zfs.misc.arcstats.prefetch_data_hits"};
	my $prefetch_data_misses = $Kstat->{"kstat.zfs.misc.arcstats.prefetch_data_misses"};
	my $prefetch_metadata_hits = $Kstat->{"kstat.zfs.misc.arcstats.prefetch_metadata_hits"};
	my $prefetch_metadata_misses = $Kstat->{"kstat.zfs.misc.arcstats.prefetch_metadata_misses"};

	my $anon_hits = $arc_hits - ($mfu_hits + $mru_hits + $mfu_ghost_hits + $mru_ghost_hits);
	my $arc_accesses_total = ($arc_hits + $arc_misses);
	my $demand_data_total = ($demand_data_hits + $demand_data_misses);
	my $prefetch_data_total = ($prefetch_data_hits + $prefetch_data_misses);
	my $real_hits = ($mfu_hits + $mru_hits);

	printf("ARC Efficiency:\t\t\t\t\t%s\n", fHits($arc_accesses_total));
	printf("\tCache Hit Ratio:\t\t%s\t%s\n",
		fPerc($arc_hits, $arc_accesses_total), fHits($arc_hits));
	printf("\tCache Miss Ratio:\t\t%s\t%s\n",
		fPerc($arc_misses, $arc_accesses_total), fHits($arc_misses));
	printf("\tActual Hit Ratio:\t\t%s\t%s\n",
		fPerc($real_hits, $arc_accesses_total), fHits($real_hits));
	print "\n";
	printf("\tData Demand Efficiency:\t\t%s\t%s\n",
		fPerc($demand_data_hits, $demand_data_total), fHits($demand_data_total));

	if ($prefetch_data_total > 0){
		printf("\tData Prefetch Efficiency:\t%s\t%s\n",
			fPerc($prefetch_data_hits, $prefetch_data_total), fHits($prefetch_data_total));
	}
	print "\n";

	print "\tCACHE HITS BY CACHE LIST:\n";
	if ( $anon_hits > 0 ){
		printf("\t  Anonymously Used:\t\t%s\t%s\n",
			fPerc($anon_hits, $arc_hits), fHits($anon_hits));
	}

	printf("\t  Most Recently Used:\t\t%s\t%s\n",
		fPerc($mru_hits, $arc_hits), fHits($mru_hits));
	printf("\t  Most Frequently Used:\t\t%s\t%s\n",
		fPerc($mfu_hits, $arc_hits), fHits($mfu_hits));
	printf("\t  Most Recently Used Ghost:\t%s\t%s\n",
		fPerc($mru_ghost_hits, $arc_hits), fHits($mru_ghost_hits));
	printf("\t  Most Frequently Used Ghost:\t%s\t%s\n",
		fPerc($mfu_ghost_hits, $arc_hits), fHits($mfu_ghost_hits));

	print "\n\tCACHE HITS BY DATA TYPE:\n";
	printf("\t  Demand Data:\t\t\t%s\t%s\n",
		fPerc($demand_data_hits, $arc_hits), fHits($demand_data_hits));
	printf("\t  Prefetch Data:\t\t%s\t%s\n",
		fPerc($prefetch_data_hits, $arc_hits), fHits($prefetch_data_hits));
	printf("\t  Demand Metadata:\t\t%s\t%s\n",
		fPerc($demand_metadata_hits, $arc_hits), fHits($demand_metadata_hits));
	printf("\t  Prefetch Metadata:\t\t%s\t%s\n",
		fPerc($prefetch_metadata_hits, $arc_hits), fHits($prefetch_metadata_hits));

	print "\n\tCACHE MISSES BY DATA TYPE:\n";
	printf("\t  Demand Data:\t\t\t%s\t%s\n",
		fPerc($demand_data_misses, $arc_misses), fHits($demand_data_misses));
	printf("\t  Prefetch Data:\t\t%s\t%s\n",
		fPerc($prefetch_data_misses, $arc_misses), fHits($prefetch_data_misses));
	printf("\t  Demand Metadata:\t\t%s\t%s\n",
		fPerc($demand_metadata_misses, $arc_misses), fHits($demand_metadata_misses));
	printf("\t  Prefetch Metadata:\t\t%s\t%s\n",
		fPerc($prefetch_metadata_misses, $arc_misses), fHits($prefetch_metadata_misses));
}

sub _l2arc_summary {
	my $l2_size = $Kstat->{"kstat.zfs.misc.arcstats.l2_size"};

	if ($l2_size == 0) {
		print "L2ARC is disabled\n";
		return;
	}

	my $l2_asize = $Kstat->{"kstat.zfs.misc.arcstats.l2_asize"};
	my $l2_abort_lowmem = $Kstat->{"kstat.zfs.misc.arcstats.l2_abort_lowmem"};
	my $l2_cksum_bad = $Kstat->{"kstat.zfs.misc.arcstats.l2_cksum_bad"};
	my $l2_evict_lock_retry = $Kstat->{"kstat.zfs.misc.arcstats.l2_evict_lock_retry"};
	my $l2_evict_reading = $Kstat->{"kstat.zfs.misc.arcstats.l2_evict_reading"};
	my $l2_feeds = $Kstat->{"kstat.zfs.misc.arcstats.l2_feeds"};
	my $l2_free_on_write = $Kstat->{"kstat.zfs.misc.arcstats.l2_free_on_write"};
	my $l2_hdr_size = $Kstat->{"kstat.zfs.misc.arcstats.l2_hdr_size"};
	my $l2_hits = $Kstat->{"kstat.zfs.misc.arcstats.l2_hits"};
	my $l2_io_error = $Kstat->{"kstat.zfs.misc.arcstats.l2_io_error"};
	my $l2_misses = $Kstat->{"kstat.zfs.misc.arcstats.l2_misses"};
	my $l2_rw_clash = $Kstat->{"kstat.zfs.misc.arcstats.l2_rw_clash"};
	my $l2_write_buffer_bytes_scanned = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_buffer_bytes_scanned"};
	my $l2_write_buffer_iter = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_buffer_iter"};
	my $l2_write_buffer_list_iter = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_buffer_list_iter"};
	my $l2_write_buffer_list_null_iter = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_buffer_list_null_iter"};
	my $l2_write_bytes = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_bytes"};
	my $l2_write_full = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_full"};
	my $l2_write_in_l2 = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_in_l2"};
	my $l2_write_io_in_progress = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_io_in_progress"};
	#my $l2_write_not_cacheable = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_not_cacheable"};
	my $l2_write_passed_headroom = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_passed_headroom"};
	#my $l2_write_pios = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_pios"};
	my $l2_write_spa_mismatch = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_spa_mismatch"};
	my $l2_write_trylock_fail = $Kstat->{"kstat.zfs.misc.arcstats.l2_write_trylock_fail"};
	my $l2_writes_done = $Kstat->{"kstat.zfs.misc.arcstats.l2_writes_done"};
	my $l2_writes_error = $Kstat->{"kstat.zfs.misc.arcstats.l2_writes_error"};
	my $l2_writes_sent = $Kstat->{"kstat.zfs.misc.arcstats.l2_writes_sent"};

	my $l2_access_total = ($l2_hits + $l2_misses);
	my $l2_health_count = ($l2_writes_error + $l2_cksum_bad + $l2_io_error);

	print "L2 ARC Summary: ";
	if ($l2_health_count > 0) {
		print "(DEGRADED)\n";
	} else { print "(HEALTHY)\n"; }

	if (not $IsOpenZFS) {
		printf("\tPassed Headroom:\t\t\t%s\n", fHits($l2_write_passed_headroom));
		printf("\tTried Lock Failures:\t\t\t%s\n", fHits($l2_write_trylock_fail));
		printf("\tIO In Progress:\t\t\t\t%s\n", fHits($l2_write_io_in_progress));
	}
	printf("\tLow Memory Aborts:\t\t\t%s\n", fHits($l2_abort_lowmem));
	printf("\tFree on Write:\t\t\t\t%s\n", fHits($l2_free_on_write));
	if (not $IsOpenZFS) {
		printf("\tWrites While Full:\t\t\t%s\n", fHits($l2_write_full));
	}
	printf("\tR/W Clashes:\t\t\t\t%s\n", fHits($l2_rw_clash));
	printf("\tBad Checksums:\t\t\t\t%s\n", fHits($l2_cksum_bad));
	printf("\tIO Errors:\t\t\t\t%s\n", fHits($l2_io_error));
	if (not $IsOpenZFS) {
		printf("\tSPA Mismatch:\t\t\t\t%s\n", fHits($l2_write_spa_mismatch));
	}
	print "\n";

	printf("L2 ARC Size: (Adaptive)\t\t\t\t%s\n", fBytes($l2_asize));
	printf("\tDecompressed Data Size:\t\t\t%s\n", fBytes($l2_size));
	printf("\tCompression Factor:\t\t\t%s\n", fRatio($l2_size, $l2_asize));
	printf("\tHeader Size:\t\t\t%s\t%s\n",
		fPerc($l2_hdr_size, $l2_size), fBytes($l2_hdr_size));
	print "\n";

	if (($l2_evict_lock_retry + $l2_evict_reading) > 0) {
		print "L2 ARC Evicts:\n";
		printf("\tLock Retries:\t\t\t\t%s\n", fHits($l2_evict_lock_retry));
		printf("\tUpon Reading:\t\t\t\t%s\n", fHits($l2_evict_reading));
		print "\n";
	}
	printf("L2 ARC Breakdown:\t\t\t\t%s\n", fHits($l2_access_total));
	printf("\tHit Ratio:\t\t\t%s\t%s\n",
		fPerc($l2_hits, $l2_access_total), fHits($l2_hits));
	printf("\tMiss Ratio:\t\t\t%s\t%s\n",
		fPerc($l2_misses, $l2_access_total), fHits($l2_misses));
	printf("\tFeeds:\t\t\t\t\t%s\n", fHits($l2_feeds));
	print "\n";

	if (not $IsOpenZFS) {
		print "L2 ARC Buffer:\n";
		printf("\tBytes Scanned:\t\t\t\t%s\n", fBytes($l2_write_buffer_bytes_scanned));
		printf("\tBuffer Iterations:\t\t\t%s\n", fHits($l2_write_buffer_iter));
		printf("\tList Iterations:\t\t\t%s\n", fHits($l2_write_buffer_list_iter));
		printf("\tNULL List Iterations:\t\t\t%s\n", fHits($l2_write_buffer_list_null_iter));
		print "\n";
	}

	print "L2 ARC Writes:\n";
	if ($l2_writes_done != $l2_writes_sent) {
		printf("\tWrites Sent: (%s)\t\t\t\t%s\n",
			"FAULTED", fHits($l2_writes_sent));
		printf("\t  Done Ratio:\t\t\t%s\t%s\n",
			fPerc($l2_writes_done, $l2_writes_sent), fHits($l2_writes_done));
		printf("\t  Error Ratio:\t\t\t%s\t%s\n",
			fPerc($l2_writes_error, $l2_writes_sent), fHits($l2_writes_error));
	} else { printf("\tWrites Sent:\t\t\t%s\t%s\n", fPerc(100), fHits($l2_writes_sent)); }
}

sub _dmu_summary {
	my $zfetch_hits = $Kstat->{"kstat.zfs.misc.zfetchstats.hits"};
	my $zfetch_misses = $Kstat->{"kstat.zfs.misc.zfetchstats.misses"};
	my $zfetch_access_total = ($zfetch_hits + $zfetch_misses);

	if ($zfetch_access_total > 0) {
		print "File-Level Prefetch:\n\n";
		printf("DMU Efficiency:\t\t\t\t\t%s\n", fHits($zfetch_access_total));
		printf("\tHit Ratio:\t\t\t%s\t%s\n",
			fPerc($zfetch_hits, $zfetch_access_total),
			fHits($zfetch_hits));
		printf("\tMiss Ratio:\t\t\t%s\t%s\n",
			fPerc($zfetch_misses, $zfetch_access_total),
			fHits($zfetch_misses));
	}
}

sub _vdev_summary {
	my $vdev_cache_delegations = $Kstat->{"kstat.zfs.misc.vdev_cache_stats.delegations"};
	my $vdev_cache_misses = $Kstat->{"kstat.zfs.misc.vdev_cache_stats.misses"};
	my $vdev_cache_hits = $Kstat->{"kstat.zfs.misc.vdev_cache_stats.hits"};
	my $vdev_cache_total = ($vdev_cache_misses + $vdev_cache_hits + $vdev_cache_delegations);

	if (($IsOpenZFS ? $Kstat->{"vfs.zfs.vdev.cache_size"} : $Kstat->{"vfs.zfs.vdev.cache.size"}) == 0) {
		printf "VDEV cache is disabled\n";
	} elsif ($vdev_cache_total > 0) {
		printf("VDEV Cache Summary:\t\t\t\t%s\n", fHits($vdev_cache_total));
		printf("\tHit Ratio:\t\t\t%s\t%s\n",
			fPerc($vdev_cache_hits, $vdev_cache_total), fHits($vdev_cache_hits));
		printf("\tMiss Ratio:\t\t\t%s\t%s\n",
			fPerc($vdev_cache_misses, $vdev_cache_total), fHits($vdev_cache_misses));
		printf("\tDelegations:\t\t\t%s\t%s\n",
			fPerc($vdev_cache_delegations, $vdev_cache_total), fHits($vdev_cache_delegations));
	}
}

sub _dataset_summary {
	if ($IsOpenZFS) {
		my @poolnames = run('zpool', 'list', '-H', '-oname');
		foreach my $poolname (@poolnames) {
			chomp($poolname);
			my @Kstat_pull = run( 'sysctl_q', "kstat.zfs.$poolname.dataset" );
			my %objset;
			foreach my $kstat (@Kstat_pull) {
				chomp $kstat;
				if ($kstat =~ m/^([^:]+):\s+(.+)\s*$/s) {
					$Kstat->{$1} = $2;
					my @fields = split(/\./, $1);
					if ($fields[5] eq "dataset_name") {
						my $dataset = $2;
						$objset{$dataset} = $fields[4];
					}
				}
			}
			foreach my $dataset (sort(keys(%objset))) {
				my $objset = $objset{$dataset};
				my $pfx = "kstat.zfs.$poolname.dataset.$objset";
				my $dataset_reads = $Kstat->{"$pfx.reads"};
				my $dataset_nread = $Kstat->{"$pfx.nread"};
				my $dataset_writes = $Kstat->{"$pfx.writes"};
				my $dataset_nwritten = $Kstat->{"$pfx.nwritten"};
				my $dataset_nunlinks = $Kstat->{"$pfx.nunlinks"};
				my $dataset_ops = $dataset_reads + $dataset_writes + $dataset_nunlinks;
				my $dataset_amount = $dataset_nread + $dataset_nwritten;
				if ($dataset_ops > 0) {
					printf("Dataset statistics for:\t%s\n\n", $dataset);
					printf("\tReads:\t\t%s\t%s\n",
						fPerc($dataset_reads, $dataset_ops), fHits($dataset_reads));
					printf("\tWrites:\t\t%s\t%s\n",
						fPerc($dataset_writes, $dataset_ops), fHits($dataset_writes));
					printf("\tUnlinks:\t%s\t%s\n",
						fPerc($dataset_nunlinks, $dataset_ops), fHits($dataset_nunlinks));
					printf("\n");
					printf("\tBytes read:\t%s\t%s\n",
						fPerc($dataset_nread, $dataset_amount), fHits($dataset_nread));
					printf("\tBytes written:\t%s\t%s\n",
						fPerc($dataset_nwritten, $dataset_amount), fHits($dataset_nwritten));
					printf("\n");
				}
			}
		}
	} else {
		printf "Per dataset statistics are not available\n";
	}
}

sub _sysctl_summary {
	return unless $usetunable;
	my @Tunable = qw(
		kern.maxusers
		vm.kmem_size
		vm.kmem_size_scale
		vm.kmem_size_min
		vm.kmem_size_max
		vfs.zfs
	);
	my %sysctl_descriptions;
	if ($show_sysctl_descriptions) {
		foreach my $tunable ( run('sysctl_q','-de', @Tunable) ){
			chomp $tunable;
			my ($name, $description) = split(/=/, $tunable, 2);
			$description = "Description unavailable" unless $description;
			$sysctl_descriptions{$name}=$description;
		}
	}
	my @tunable = run('sysctl_q', '-e', @Tunable);
	print "ZFS Tunables (sysctl):\n";
	foreach my $tunable (@tunable){
		chomp($tunable);
		my ($name, $value) = split(/=/, $tunable, 2);
		my $typefmt = looks_like_number($value) ? "%d" : "%s";
		my $format = $alternate_sysctl_layout ? "\t%s=$typefmt\n" : "\t%-40s$typefmt\n";
		if ($show_sysctl_descriptions != 0){
			printf("\t\# %s\n", $sysctl_descriptions{$name});
		}
		printf($format, $name, $value);
	}
}

my @unSub = map { $_, \&div2 }(
	\&_system_info,
	\&_system_memory,
	\&_arc_summary,
	\&_arc_efficiency,
	\&_l2arc_summary,
	\&_dataset_summary,
	\&_dmu_summary,
	\&_vdev_summary,
	\&_sysctl_summary,
);

my %opt;
GetOptions( \%opt,
	qw"A D E I|F L M O R|B Z a d t u s",
	h => \&_usage,   # exits
	V => \&_version, # exits
	qw"hostname=s user=s id_file=s port=s",
) || _usage;

for( grep { length > 1 } keys %opt ){
	$ssh{$_} = $opt{$_}
}

$alternate_sysctl_layout = 1 if $opt{t};
$show_sysctl_descriptions = 1 if $opt{d};
$raw_data = 1 if $opt{R};

my @Kstat_pull = run( 'sysctl_q', @Kstats );
foreach my $kstat (@Kstat_pull) {
	chomp $kstat;
	if ($kstat =~ m/^([^:]+):\s+(.+)\s*$/s) {
		$Kstat->{$1} = $2;
	};
};

if (!$Kstat->{"vfs.zfs.version.spa"}) {
	printf(STDERR "ZFS module not loaded\n");
	exit 1;
};

my @out;
if ($opt{a}) {
	@out = @unSub;
} else {
	if ($opt{I}) { push @out, \&_system_info, \&div2 }
	if ($opt{M}) { push @out, \&_system_memory, \&div2 }
	if ($opt{A}) { push @out, \&_arc_summary, \&div2 }
	if ($opt{E}) { push @out, \&_arc_efficiency, \&div2 }
	if ($opt{L}) { push @out, \&_l2arc_summary, \&div2 }
	if ($opt{O}) { push @out, \&_dataset_summary, \&div2 }
	if ($opt{Z}) { push @out, \&_dmu_summary, \&div2 }
	if ($opt{D}) { push @out, \&_vdev_summary, \&div2 }
	if ($opt{s}) { push @out, \&_sysctl_summary, \&div2 }
}
if (!@out) { _usage; exit; }
_start;
$_->() for @out;

__END__
