# SPDX-License-Identifier: CDDL-1.0
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or https://opensource.org/licenses/CDDL-1.0.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/redundancy/redundancy.cfg

function cleanup
{
	if poolexists $TESTPOOL; then
		destroy_pool $TESTPOOL
	fi
	typeset dir
	for dir in $TESTDIR $BASEDIR; do
		if [[ -d $dir ]]; then
			log_must rm -rf $dir
		fi
	done
}

#
# Get the number of checksum errors for the pool.
#
# $1 Pool
#
function cksum_pool
{
	typeset -i cksum=$(zpool status -p $1 | awk '
	    !NF { isvdev = 0 }
	    isvdev { errors += $NF }
	    /CKSUM$/ { isvdev = 1 }
	    END { print errors }
	')

	echo $cksum
}

#
# Record the directories construction and checksum all the files which reside
# within the specified pool
#
# $1 The specified pool
# $2 The file which save the record.
#
function record_data
{
	typeset pool=$1
	typeset recordfile=$2

	[[ -z $pool ]] && log_fail "No specified pool."
	[[ -f $recordfile ]] && log_must rm -f $recordfile

	sync_pool $pool
	typeset mntpnt
	mntpnt=$(get_prop mountpoint $pool)
	log_must eval "du -a $mntpnt > $recordfile 2>&1"
	#
	# When the data was damaged, checksum is failing and return 1
	# So, will not use log_must
	#
	find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1
}

#
# Create test pool and fill with files and directories.
#
# $1 pool name
# $2 pool type
# $3 virtual devices number
#
function setup_test_env
{
	typeset pool=$1
	typeset keyword=$2
	typeset -i vdev_cnt=$3
	typeset vdevs

	typeset -i i=0
	while (( i < vdev_cnt )); do
		vdevs="$vdevs $BASEDIR/vdev$i"
		((i += 1))
	done

	if [[ ! -d $BASEDIR ]]; then
		log_must mkdir $BASEDIR
	fi

	if poolexists $pool ; then
		destroy_pool $pool
	fi

	log_must truncate -s $MINVDEVSIZE $vdevs

	log_must zpool create -O compression=off -f -m $TESTDIR $pool $keyword $vdevs

	log_note "Filling up the filesystem ..."
	typeset -i i=0
	typeset file=$TESTDIR/file
	typeset -i limit
	(( limit = $(get_prop available $pool) / 2 ))

	while true ; do
		[[ $(get_prop available $pool) -lt $limit ]] && break
		file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES || break
		(( i = i + 1 ))
	done

	record_data $TESTPOOL $PRE_RECORD_FILE
}

function refill_test_env
{
	log_note "Re-filling the filesystem ..."
	typeset pool=$1
	typeset -i i=0
	typeset mntpnt
	mntpnt=$(get_prop mountpoint $pool)
	typeset file=$mntpnt/file
	while [[ -e $file.$i ]]; do
		log_must rm -f $file.$i
		file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES || break
		(( i = i + 1 ))
	done

	record_data $TESTPOOL $PRE_RECORD_FILE
}

#
# Check pool status is healthy
#
# $1 pool
#
function is_healthy
{
	typeset pool=$1

	typeset healthy_output="pool '$pool' is healthy"
	typeset real_output=$(zpool status -x $pool)

	if [[ "$real_output" == "$healthy_output" ]]; then
		return 0
	else
		typeset -i ret
		zpool status -x $pool | grep "state:" | grep -q "FAULTED" && return 1
		typeset l_scan
		typeset errnum _
		l_scan=$(zpool status -x $pool | grep "scan:")
		l_scan=${l_scan##*"with"}
		read -r errnum _ <<<"$l_scan"

		return $errnum
	fi
}

#
# Check pool data is valid
#
# $1 pool
#
function is_data_valid
{
	typeset pool=$1

	log_must zpool scrub -w $pool

	record_data $pool $PST_RECORD_FILE
	if ! cmp $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null; then
		log_must cat $PRE_RECORD_FILE
		log_must cat $PST_RECORD_FILE
		diff -u $PRE_RECORD_FILE $PST_RECORD_FILE
		return 1
	fi

	return 0
}

#
# Get the specified count devices name
#
# $1 pool name
# $2 devices count
#
function get_vdevs #pool cnt
{
	typeset pool=$1
	typeset -i cnt=$2

	typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}' | \
		grep -vEe "^pool$|^capacity$|^mirror\-[0-9]$|^raidz[1-3]\-[0-9]$|^draid[1-3].*\-[0-9]$|---" \
		-e "/old$|^$pool$")
	typeset -i i=0
	typeset vdevs
	while ((i < cnt)); do
		typeset dev _
		read -r dev _ <<<"$all_devs"
		eval all_devs=\${all_devs##*$dev}

		vdevs="$dev $vdevs"
		((i += 1))
	done

	echo "$vdevs"
}

#
# Create and replace the same name virtual device files
#
# $1 pool name
# $2-n virtual device files
#
function replace_missing_devs
{
	typeset pool=$1
	shift

	typeset vdev
	for vdev in $@; do
		log_must dd if=/dev/zero of=$vdev \
		    bs=1024k count=$((MINVDEVSIZE / (1024 * 1024))) \
		    conv=fdatasync
		log_must zpool replace -wf $pool $vdev $vdev
	done
}

#
# Damage the pool's virtual device files.
#
# $1 pool name
# $2 Failing devices count
# $3 damage vdevs method, if not null, we keep
#    the label for the vdevs
#
function damage_devs
{
	typeset pool=$1
	typeset -i cnt=$2
	typeset label="$3"
	typeset vdevs
	typeset -i bs_count=$(((MINVDEVSIZE / 1024) - 4096))

	vdevs=$(get_vdevs $pool $cnt)
	typeset dev
	if [[ -n $label ]]; then
		for dev in $vdevs; do
			log_must dd if=/dev/zero of=$dev seek=512 bs=1024 \
			    count=$bs_count conv=notrunc >/dev/null 2>&1
		done
	else
		for dev in $vdevs; do
			log_must dd if=/dev/zero of=$dev bs=1024 \
			    count=$bs_count conv=notrunc >/dev/null 2>&1
		done
	fi

	sync_pool $pool
}

#
# Clear errors in the pool caused by data corruptions
#
# $1 pool name
#
function clear_errors
{
	typeset pool=$1

	log_must zpool clear $pool

	if ! is_healthy $pool ; then
		log_note "$pool should be healthy."
		return 1
	fi
	if ! is_data_valid $pool ; then
		log_note "Data should be valid in $pool."
		return 1
	fi

	return 0
}

#
# Remove the specified pool's virtual device files
#
# $1 Pool name
# $2 Missing devices count
#
function remove_devs
{
	typeset pool=$1
	typeset -i cnt=$2
	typeset vdevs

	vdevs=$(get_vdevs $pool $cnt)
	log_must rm -f $vdevs

	sync_pool $pool
}

#
# Recover the bad or missing device files in the pool
#
# $1 Pool name
# $2 Missing devices count
#
function recover_bad_missing_devs
{
	typeset pool=$1
	typeset -i cnt=$2
	typeset vdevs

	vdevs=$(get_vdevs $pool $cnt)
	replace_missing_devs $pool $vdevs

	if ! is_healthy $pool ; then
		log_note "$pool should be healthy."
		return 1
	fi
	if ! is_data_valid $pool ; then
		log_note "Data should be valid in $pool."
		return 1
	fi

	return 0
}

#
# Given a dRAID pool issue a scrub and verify the current pool status
# aligns with the expected status based on the 'replace_mode' passed.
# Valid modes are:
#
# 1. healing - The pool is perfectly intact.  No checksum errors have
#    been reported and the scrub didn't make any repairs.  This is the
#    expected state after a healing resilver of a healthy pool.
#
# 2. sequential - The pool is fully intact.  There should never be a
#    checksum error.
#
# 3. damaged - The pool was intentionally silently damaged.  Checksum
#    errors are expected to be reported as the damaged blocks are
#    detected and repaired.
#
# In all of these cases a scrub must be able to successfully repair the
# pool and result in no data loss.
#
function verify_draid_pool
{
	typeset pool=${1:-$TESTPOOL}
	typeset replace_mode=${2:-healing}

	log_note "verify_draid_pool $pool $replace_mode"
	log_must zpool scrub -w $pool

	typeset -i cksum=$(cksum_pool $pool)

	if [[ "$replace_mode" = "healing" ]]; then
		if [[ $cksum -gt 0 ]]; then
			log_must zpool status -v $pool
			log_fail "Unexpected CKSUM errors found for $pool ($cksum)"
		fi

		if ! check_pool_status $pool "scan" "repaired 0B"; then
			log_must zpool status -v $pool
			log_fail "Unexpected repair IO found for $pool ($cksum)"
		fi
	elif [[ "$replace_mode" = "sequential" ]]; then
		if [[ $cksum -gt 0 ]]; then
			log_must zpool status -v $pool
			log_fail "Unexpected CKSUM errors found for $pool ($cksum)"
		fi
	elif [[ "$replace_mode" = "damaged" ]]; then
		if [[ $cksum -lt 1 ]]; then
			log_must zpool status -v $pool
			log_fail "Expected CKSUM errors missing for $pool ($cksum)"
		fi

		if check_pool_status $pool "scan" "repaired 0B"; then
			log_must zpool status -v $pool
			log_fail "Expected repair IO missing for $pool ($cksum)"
		fi
	else
		log_fail "Invalid replace_mode=$replace_mode"
	fi

	if ! check_pool_status $pool "scan" "with 0 errors"; then
		log_must zpool status -v $pool
		log_fail "Unexpected repair errors found for $pool"
	fi

	if ! check_pool_status $pool "errors" "No known data errors"; then
		log_must zpool status -v $pool
		log_fail "Unexpected data errors found for $pool"
	fi
}
