#!/opt/local/bin/perl
#
#  Name: get_bd_poros.pl -- Determine bulk density and porosity
#
#  Purpose:
#	This script determines bulk density (BD) and porosity values for
#	each of a set of standard-thickness soil layers for each STATSGO
#	mapunit in a state.
#
#	Bulk density is the defined as the total mass of soil per unit
#	volume.  It is determined from the STATSGO Layer table variables
#	BDH and BDL.
#
#	Porosity is defined as the total volume occupied by pores per
#	unit volume of soil and is calculated from bulk density and
#	particle density.  The formula is:  Porosity = 1 - (BD/PD),
#	where PD is the particle density, normally assumed to be 2.65
#	g/cm^3.  It should be noted that porosity is a derived quantity
#	and is NOT a part of the original STATSGO database.
#
#  Usage:
#	perl get_bd_poros.pl layerdeftab workspace outfile rptfile
#
#  Options and Arguments:
#	layerdeftab	File specification for table defining
#			thicknesses of standard soil layers.
#	workspace	Arc/Info Workspace location of INFO tables.
#	outfile		File to receive output table.
#	rptfile		File to receive report of processing statistics,
#			problems, etc.
#
#  External associations:
#	This script is part of a package for processing the STATSGO soil
#	properties database.  Output from this script is in an ASCII
#	format; script pack_bd_poros.pl may be used to convert it to a
#	format suitable for ingest into Arc/Info INFO tables defined by
#	scripts def_bd_items.aml and def_poros_items.aml.
#
#  Algorithmic details:
#	The STATSGO Layer table gives a range of bulk density values
#	(BDH and BDL) for each layer of each component of each mapunit.
#	The mean bulk density for each layer is computed as the simple
#	average of BDH and BDL.  The corresponding porosity is computed
#	as 1 - (BD/PD), using a particle density PD = 2.65.  The BD and
#	porosity values for each component are then interpolated from
#	the layer depths of the particular component (specified by Layer
#	table variables LAYDEPL and LAYDEPH) to the standard layers
#	specified by the input table of standard layers.  The BD and
#	porosity values for each standard layer are then averaged over
#	all non-water components of the mapunit, weighting the
#	contribution of each component by the COMPPCT variable from the
#	STATSGO Component table.
#
#	The region from the bottom of the last component layer to the
#	bottom of the last standard layer, if any, is assumed to be the
#	same as the lowest component layer down to the mean bedrock
#	depth (mean of Component table variables ROCKDEPL and
#	ROCKDEPH).  Below this depth, BD is set equal to the particle
#	density, PD = 2.65 g/cm^3, and the porosity is set to zero.
#	
#	NOTE that many components specify ROCKDEPL = ROCKDEPH = 60
#	inches to specify that the soil was not examined below this
#	depth.  In most cases, bedrock is not actually present;
#	nonetheless, the values of BD and porosity are computed as if
#	bedrock was encountered at this depth.
#
#	A number of non-water components, and in some cases entire
#	mapunits, have BDL=BDH=0 or no entry for these values.  These
#	components are ignored in the computation.  If all non-water
#	components for the mapunit have BD=0, the BD value is set to 0
#	and the porosity to 1.01.  If the mapunit is entirely water, the
#	porosity is set to 1.00 and BD to zero.
#
#  External files accessed:
#	<layerdeftab>	READ to get thickness of each standard
#			layer. This file contains one record for each
#			layer, starting at the top, giving its thickness
#			in cm.
#	<workspace>	Arc/Info Workspace location of INFO tables being 
#			accessed for processing 
#	<outfile>	WRITE to record BD and porosity for each
#			mapunit.  Each record contains the mapunit ID,
#			the BD values for each standard layer, and the
#			porosity for each standard layer.
#	<rptfile>	WRITE to record processing arguments, statistics
#			and other problems, etc., encountered during
#			processing.
#
#  External subroutines included:
#	Openinfo.pl	Open an INFO table, returning unpack() template
#			for reading a specified list of items.
#
#  Internal subroutines defined:
#	help		Output help text.
#	bld_layers	Subroutine to build layers for one mapunit
#			component.
#	output_mu	Generate output values for one mapunit, and
#			reinitialize for next component.
#	set_allrock	Subroutine to build single rock or water layer
#			when no soil.
#	usage		Output terse usage text.
#
#  Internal variables:
#	COMP		File handle for reading Component table.
#	LAYDEF		File handle for reading layer definition table.
#	LAYER		File handle for reading Layer table.
#	PDEN		Particle density of rock.
#	RPT		File handle for output report file.
#	UCONV		Conversion factor from STATSGO depth units to
#			standard depth units.
#	bufc		Array of values from COMP table.
#	bufl		Array of values from LAYER table.
#	cl		Current component layer.
#	clbd		Array of component layer bulk densities.
#	clbot		Array of component layer bottoms, in cm.
#	cldeph		Bottom of component layer, LAYDEPH item.
#	cldepl		Top of component layer, LAYDEPL item.
#	clporos		Array of component layer porosities.
#	cltop		Array of component layer tops, in cm.
#	cmd		Name of this script; used for help text.
#	compname	Component name from COMP table.
#	compnonzbd	Flag set if component has at least one layer
#			with BD > 0 or is all water.
#	comppct		Percent of component in mapunit from COMP table.
#	comprec		Number of current COMP record.
#	curmuid		ID of mapunit currently being processed.
#	errcnt		Count of severe errors.
#	lbd		Array of bd accumulators for each standard
#			output layer.
#	layerdeftab	Filename for table defining standard layers.
#	layer		Number of current output layer.
#	laynum		Value of LAYERNUM item.
#	layerrec	Number of current LAYER record
#	lmuid		Mapunit ID of current layer.
#	lporos		Array of porosity accumulators for each standard
#			output layer.
#	lseqnum		Component sequence number of current layer.
#	lthick		Array of thicknesses of standard layers.
#	ltop		Array of depths to tops of standard layers.
#	muid		Mapunit ID.
#	munonzbd	Flag indicating at least one non-water component 
#			with one or more levels for which BD > 0.
#	muzerobd	Flag indicating at least one non-water component
#			with BD = 0 for all levels.
#	ncl		Number of layers for current component.
#	nextcl		Next component layer.
#	nlayer		Number of standard layers.
#	nonztotpct	Sum of percents for components with BD > 0.
#	opt		Name of current option.
#	outfile		Name of file to receive relayered output.
#	overlap		Amount of overlap of component and output layers.
#	rdeph		Bedrock depth for component, higher abs. value.
#	rdepl		Bedrock depth for component, lower abs. value.
#	rdepm		Bedrock depth for component, mean.
#	reclen_comp	Length of COMP table record, in bytes.
#	reclen_layer	Length of LAYER table record, in bytes.
#	savecl		Array to save data for next component layer.
#	seqnum		Component sequence number.
#	stats		Associative array of statistics for anomalies.
#	template_comp	Template for reading COMP table, returned by openinfo
#	template_lay	Template for reading LAYER table, returned by openinfo
#	time		Array of values returned by localtime().
#	totpct		Sum of percents for all components in map unit.
#	value		Value of one array element.
#
#  Script history:
#
#	 9-96	Initial version for bulk-density only, D. A. Miller
#	11-96	Added flags for tracking map unit components with BD = 0
#		for all layers.
#	12-96	Merged with porosity computation from get_pore.pl
#		script.
#
############   End of Prolog for get_bd_poros.pl   ############

#  Define usage and help text

@cmd =	(split ('/', $0));
$cmd = pop (@cmd);

require "openinfo.pl";

sub usage
    {
    print "\n";
    print "Usage:  perl $cmd layerdeftab workspace outfile rptfile\n"; 
    print "        perl $cmd, -u|h\n";

    }		#  End of subroutine usage

sub help
    {
    print "\n";
    print $cmd, " -- Get BD and porosity for each STATSGO mapunit\n";
    &usage;
    print "\n";
    print "Options and arguments are\n";
    print "      -h	Write this help text to stdout and exit.\n";
    print "      -u	Display terse usage message and exit.\n";   
    print "  layerdeftab	File specification for table defining\n",
	  "		thicknesses of standard soil layers.\n";
    print "   workspace	Arc/Info workspace location of INFO tables.\n";
    print "    outfile	File to receive output table.\n",
	  "    rptfile	File to receive report of processing",
			" statistics,\n",
	  "		problems, etc.\n";
    }		#  End of subroutine help

#
#  Subroutine to build layers for one mapunit component
#

sub bld_layers
    {
    $stats{"totalcomp"}++;	# Total components
    $ncl = $nextcl;
    return 1 if (($ncl == 0) && ( ! &set_allrock));
    $totpct += $comppct;

#  Convert bedrock depths to cm

    if (($rdepl eq "") || ($rdeph eq ""))
	{
	print RPT "Mapunit $curmuid, component $seqnum:	no ROCKDEP",
	  " specified\n" unless ($clporos[0] <= 0.0);
	$rdepl = 0 if ($rdepl eq "");
	$rdeph = 0 if ($rdeph eq "");
	}
    $rdepl *= $UCONV;
    $rdeph *= $UCONV;
    $rdepm = ($rdepl + $rdeph) / 2.0;

#  Check whether component specifies non-zero BD

    for ($cl = 0; $cl < $ncl; $cl++)
	{
	if ($clbd[$cl] > 0.0)
	    {
	    $compnonzbd = 1;
	    }
	elsif (($cltop[$cl] >= $rdepm) && ($clporos[$cl] > 0.0))
	    {
	    $clbd[$cl] = $PDEN;
	    $clporos[$cl] = 0.0;
	    $compnonzbd = 1 if ($cl == 0);
	    }
	}
    if ($compnonzbd)
	{
	$nonztotpct += $comppct;
	$munonzbd = 1;
	}
    else
	{
	$muzerobd = 1 unless ($clporos[0] == 0.0);
	return 1;
	}
  
#  Check for layer overlaps, inversion, etc.

    if ($cltop[0] != 0.0)
	{
	print RPT "Layer 1 of $curmuid $seqnum starts at $cltop[0]",
	  " -- changed to 0.0\n";
	$cltop[0] = 0.0;
	$stats{"nonzero"}++;
	}
    for ($cl = 0; $cl < $ncl; $cl++)
	{
	if ($cltop[$cl] > $clbot[$cl])
	    {
	    print RPT "LAYDEPH = $clbot[$cl] less than LAYDEPL =",
	      " $cltop[$cl], $curmuid $seqnum, layer $cl\n";
	    $clbot[$cl] = $cltop[$cl];
	    die "Too many errors -- see report file $rptfile: "
	      if ($errcnt++ > 10);
	    }	    
	if (($cl > 0) && ($cltop[$cl] < $clbot[$cl - 1]))
	    {
	    if (($cltop[$cl] == $cltop[$cl -1])
	      && ($clbot[$cl] == $clbot[$cl - 1]))
		{
		$value = $cl + 1;
		print RPT "Layer $value duplicates $cl, $curmuid",
		  " $seqnum -- using first only\n";
		$cltop[$cl] = $clbot[$cl];
		$stats{"duplayer"}++;
		}
	    elsif ($cltop[$cl] > $cltop[$cl - 1])
		{
		$value = $cl + 1;
		print RPT "Layer $value overlaps layer $cl, $curmuid",
		  " $seqnum -- moving top of layer $value downwards\n";
		$cltop[$cl] = $clbot[$cl - 1];
		$clbot[$cl] = $cltop[$cl]
		  if ($clbot[$cl] < $cltop[$cl]); 
		$stats{"overlap"}++;
		}
	    else
		{
		$value = $cl + 1;
		print RPT "Layer $value starts above $cl, $curmuid",
		  " $seqnum -- moving layer $value downwards\n";
		$cltop[$cl] = $clbot[$cl - 1];
		$clbot[$cl] = $cltop[$cl]
		  if ($clbot[$cl] < $cltop[$cl]); 
		die "Too many errors -- see report file $rptfile: "
		  if ($errcnt++ > 10);
		}
	    }
	elsif (($cl > 0) && ($cltop[$cl] > $clbot[$cl - 1]))
	    {
	    $value = $cl + 1;
	    print RPT "Gap between layer $cl and layer $value,",
	      " $curmuid $seqnum -- extending layer $cl downwards\n";
	    $clbot[$cl - 1] = $cltop[$cl];
	    $stats{"layergap"}++;
	    }
	}

#  Extend component layer coverage to bottom of output layers

    if ($clbot[$ncl - 1] < $ltop[$nlayer])
	{
	$stats{"lextend"}++;	# Component layer extended downwards
	if ($clbot[$ncl - 1] >= $rdepm)
	    {
	    $cltop[$ncl] = $clbot[$ncl - 1];
	    $clbot[$ncl] = $ltop[$nlayer];
	    $clbd[$ncl] = $PDEN;
            $clporos[$ncl++] = 0.0;
	    }
	else
	    {
	    $clbot[$ncl - 1] = $rdepm;
	    if ($ltop[$nlayer] > $rdepm)
		{
		$cltop[$ncl] = $clbot[$ncl - 1];
		$clbot[$ncl] = $ltop[$nlayer];
		$clbd[$ncl] = $PDEN;
		$clporos[$ncl++] = 0.0;
		}
	    }
	}
    
#  Get bulk density and porosity contributions to output layers

    $cl = 0;
    for ($layer = 0; $layer < $nlayer; $layer++)
	{
	$overlap = $lthick[$layer];
	for ( ; $clbot[$cl] < $ltop[$layer + 1]; $cl++)
	    {
	    $overlap -= ($ltop[$layer + 1] - $clbot[$cl]);
	    $lbd[$layer]
	      += $comppct * $clbd[$cl] * ($overlap / $lthick[$layer]); 
            $lporos[$layer]
              += $comppct * $clporos[$cl] * ($overlap / $lthick[$layer]);
	    $overlap = $ltop[$layer + 1] - $clbot[$cl];
	    }
	$lbd[$layer]
	  += $comppct * $clbd[$cl] * ($overlap / $lthick[$layer]); 
        $lporos[$layer]
              += $comppct * $clporos[$cl] * ($overlap / $lthick[$layer]); 
	}
    $compnonzbd = 0; 
    return 1;
      
    }		# End of subroutine bld_layers

#
#  Subroutine to output layer data for one mapunit
#

sub output_mu
    {
    return $muid if ($curmuid eq "");

#  Report problems and anomalies

    print RPT "Total COMPPCT for MUID $curmuid = $totpct\n"
      if (($totpct < 99) || ($totpct > 101 ));

    if ($muzerobd)
       {
       if ($munonzbd)
	  {
#	  print RPT "Mapunit $curmuid has BDL=BDH=0 for some non-water",
#	   " components\n";
	  $stats{"somezbd"}++;
	  }
       else
	  {
	  print RPT "Mapunit $curmuid has BDH=BDL=0 for all non-water",
	   " components\n";
	  $stats{"allzbd"}++;
	  }
       }

#  Write record for this mapunit

    printf OUT "%-8s", $curmuid;
    if ($nonztotpct <= 0)
	{
	$nonztotpct = 100;
	for ($layer = 0; $layer < $nlayer; $layer++)
	    {
	    $lbd[$layer] = 0;
	    $lporos[$layer] = ($muzerobd) ? 101 : 100;
	    }
	}
    for ($layer = 0; $layer < $nlayer; $layer++)
	{
	printf OUT "%5.2f",($lbd[$layer]/$nonztotpct);
	$lbd[$layer] = 0.0;
	}
    for ($layer = 0; $layer < $nlayer; $layer++)
	{
	printf OUT "%5.2f",($lporos[$layer]/$nonztotpct);
	$lporos[$layer] = 0.0;
	}
    print OUT "\n";

#  Reinitialize accumlators not done above

    $totpct = 0;
    $nonztotpct = 0;
    $munonzbd = 0;
    $muzerobd = 0; 
   
    return $muid;
    }			#  End of subroutine output_mu

#
#  Subroutine to build single rock layer when no soil
#

sub set_allrock
    {
    if (($compname =~ /^\s*WATER\s*$/) || ($curmuid =~ /^[A-Z]{2}W/))
	{
	$nextcl = $ncl = 1;
	$cltop[0] = 0;
	$clbot[0] = $ltop[$nlayer];
	$clbd[0] = 0.0;
	$clporos[0] = 0.0;
	}
    
    elsif ($rdeph == 0)
	{
	print RPT "No layers for $curmuid $seqnum -- assuming",
	  " solid rock\n";
	$nextcl = $ncl = 1;
	$cltop[0] = 0;
	$clbot[0] = $ltop[$nlayer];
	$clbd[0] = $PDEN;
	$clporos[0] = 0.0;
	}
    else
	{
	print RPT "MUID/SEQNUM mismatch -- ",
	  "compfile:  $curmuid $seqnum,	 ",
	  "layerfile:  $lmuid $lseqnum\n";
	die "Too many errors -- see report file $rptfile: "
	  if ($errcnt++ > 10);
	return 0;
	}
    return 1;
    }

			#  End of subroutine set_allrock

#  Start of main routine
#

#  Get options

while ($ARGV[0] =~ /-(\w+)(\s+|$)/)
    {
    push (@opt, $1);
    shift;
    }	
if ($opt[0] eq 'u')
    {
    &usage;
    exit;
    }
if ($opt[0] eq 'h')
    {
    &help;
    exit;
    }

#  Preprocess arguments

if  ($#ARGV < 3)
    {
    &usage;
    exit 1;
    }

$layerdeftab = shift;
$workspace = shift;
$outfile = shift;
$rptfile = shift;

# Set program constants

$UCONV = 2.54;	 # Convert inches to centimeters
$PDEN = 2.65;	 # Particle density in g^cm3.

# Open Layer defintion file;  Get layer depths.

$nlayer = 0;
$ltop[0] = 0;

open (LAYDEF, $layerdeftab)
  || die "Error opening layer table, $layerdeftab,";
while (<LAYDEF>)
    {
    if (/^\s*(\d+)($|\s+)/)
	{
	$lthick[$nlayer++] = $1;
	$ltop[$nlayer] = $ltop[$nlayer - 1] + $1;
	}
    }
close LAYDEF;

#  Open	 output files

open (OUT, "> $outfile") || die "Error opening output file, $outfile,";
open (RPT, "> $rptfile") || die "Error opening report file, $rptfile,";

#  Create report header

@time = localtime (time);
$time[5] += 1900;
printf RPT "	%s, %02d/%02d/%02d %02d:%02d:%02d\n\n",
  "Summary of get_bd_poros.pl processing", $time[5], $time[4]+1,
  $time[3], $time[2], $time[1];

print RPT "Arguments are:\n";

printf RPT "	%-16s%s\n", "layerdeftab", $layerdeftab;
printf RPT "	%-16s%s\n", "workspace", $workspace;
printf RPT "	%-16s%s\n", "outfile", $outfile;
printf RPT "	%-16s%s\n\n", "rptfile", $rptfile;

#  Call openinfo to open and unpack the INFO formatted COMP table

(($reclen_comp, $template_comp) = &openinfo ("COMP",$workspace, "COMP",
    "MUID", "SEQNUM", "COMPNAME", "COMPPCT", "ROCKDEPL", "ROCKDEPH"))
  || die "Openinfo returned error";

#  Call openinfo to open and unpack the INFO formatted LAYER table

(($reclen_layer, $template_lay) = &openinfo ("LAYER", $workspace,
    "LAYER", "MUID", "SEQNUM", "LAYERNUM", "LAYDEPL", "LAYDEPH", "BDL",
    "BDH"))
  || die "Openinfo returned error";

#  Clear all bd and porosity entries

for ($layer = 0; $layer < $nlayer; $layer++)
    {
    $lbd[$layer] = 0.0;
    $lporos[$layer] = 0.0;
    }
   
#  Process components of each mapunit in turn

$curmuid = "";
$totpct = 0;
$nonztotpct = 0;
$nextcl = 0;
$comprec = 0;
$layerrec = 0;
$compnonzbd = 0;	
$munonzbd = 0;
$muzerobd = 0;

while (read (COMP, $bufc, $reclen_comp))
    {
    $comprec++;
    (($muid, $seqnum, $compname, $comppct, $rdeph, $rdepl)
        = unpack ($template_comp, $bufc))
      || die "Bad COMP record $comprec";
    $curmuid = &output_mu if ($muid ne $curmuid);

 #  Collect layer data for current component

    if (defined $laynum)
	{
	print RPT "Bad LAYERNUM \"$laynum\" for layer 1,",
	    " $lmuid $lseqnum\n"
	  if (($laynum != 1) && ( ! eof (LAYER)));
	if (($lmuid ne $curmuid) || ($lseqnum != $seqnum))
	    {
	    &bld_layers if (&set_allrock);
	    ($cltop[0], $clbot[0], $clbd[0], $clporos[0]) = @savecl;
	    next;
	    }
	}
    while (read (LAYER, $bufl, $reclen_layer))
	{
	$layerrec++;
	(($lmuid, $lseqnum, $laynum, $cldepl, $cldeph, $bdl, $bdh)
	    = unpack ($template_lay, $bufl))
	  || die "Bad LAYER record $layerrec";
	$cltop[$nextcl] = $cldepl * $UCONV;
	$clbot[$nextcl] = $cldeph * $UCONV;

#  Compute bulk density for this component layer

	if ($bdl > $bdh)
	    {
	    print RPT "BDL > BDH, $lmuid $lseqnum, layer $laynum",
	      " -- layer ignored\n";
	    $stats{"valrev"}++;
	    die "Too many errors -- see report file $rptfile: "
	      if ($errcnt++ > 10);
	    next;
	    }

	$clbd[$nextcl] = (($bdl + $bdh)/2.0);
        $clporos[$nextcl] = 1 - ($clbd[$nextcl]/$PDEN);

	last if (($lseqnum != $seqnum) || ($lmuid ne $curmuid));
 
	print RPT "Bad LAYERNUM \"$laynum\" for layer",
	  " $nextcl, $lmuid $lseqnum\n"
	  if (++$nextcl != $laynum);
	}
    @savecl = ($cltop[$nextcl], $clbot[$nextcl], $clbd[$nextcl],
      $clporos[$nextcl]);
    $nextcl = &bld_layers;
    ($cltop[0], $clbot[0], $clbd[0], $clporos[0]) = @savecl;
    }

&output_mu;		# Output results for last mapunit

close COMP;
close LAYER;
close OUT;

#  Add statistics to processing report.

print RPT "\nProcessing statistics\n\n";
$value = 0 unless ($value = $stats{"totalcomp"});
print RPT "    Total components in input compfile:          $value\n"; 
print RPT "    Components without valid bulk density:       $value\n"
  if ($value = $stats{"novalue"});
print RPT "    Mapunits with BD = 0 for all components:     $value\n"
  if ($value = $stats{"allzbd"});
print RPT "    Mapunits with BD = 0 for some components:    $value\n"
  if ($value = $stats{"somezbd"});
close RPT;

exit 0;		# From Script get_bd_poros.pl
