fields_local

This module implements the same implicit algorithm as fields_implicit. However, it is an almost complete rewrite of the implementation which can give large gains in the speed of calculating the response matrix, as well as significant gains to the advance steps

Uses

- mp

Variables

dlun do_smart_update debug initialised reinit dump_response read_response field_local_allreduce field_local_allreduce_sub field_local_tuneminnrow field_local_nonblocking_collectives nfield minNRow pc fieldmat condition_number_warning_limit

Derived Types

fields_local_testing rowblock_type cell_type supercell_type ky_type fieldmat_type pc_type

Functions

rb_mv_mult_fun rb_irex_to_irloc c_has_row sc_has_it sc_get_left_it ky_is_from_it pc_current_nresp pc_has_ik pc_has_it fields_local_functional l2norm

Subroutines

rb_allocate rb_deallocate rb_init rb_debug_print rb_mv_mult rb_set_nrow rb_reset c_allocate c_deallocate c_init c_debug_print c_mv_mult_rb c_get_field_update c_reset c_set_locality sc_allocate sc_deallocate sc_init sc_debug_print sc_get_field_update sc_reduce_tmpsum sc_iex_to_dims sc_iex_to_ifl sc_iex_to_ic sc_iex_to_it sc_iex_to_ig sc_reset sc_set_locality sc_dump sc_dump_to_file sc_read_from_file sc_store_fq sc_prepare sc_invert sc_invert_local sc_invert_mpi sc_pull_rows_to_arr sc_push_arr_to_rows sc_make_subcom_1 sc_make_subcom_2 ky_allocate ky_deallocate ky_init ky_debug_print ky_get_field_update ky_wait_for_nonblocking_collectives ky_store_fq ky_reset ky_set_locality ky_prepare ky_make_subcom_1 ky_make_subcom_2 tuneMinNRow fm_allocate fm_deallocate fm_debug_print fm_set_is_local fm_init fm_populate fm_init_next_field_points fm_prepare fm_get_field_update fm_unpack_to_field fm_reset fm_count_subcom fm_make_subcom_1 fm_make_subcom_2 fm_calc_sc_heads fm_gather_fields fm_update_fields fm_update_fields_newstep fm_get_condition_numbers fm_write_debug_data fm_getfieldeq_nogath fm_reduce_an_head_send_broadcast fm_check_an fm_getfieldeq1_nogath pc_allocate pc_deallocate pc_debug_print pc_find_locality pc_count_avail pc_make_decomp pc_reset pc_init pc_decomp_all_serial_local pc_decomp_own_serial_local pc_decomp_owncells_serial_local pc_decomp_owncells_simplempi init_fields_local init_fields_matrixlocal reset_fields_local finish_fields_local getfield_local init_allfields_local advance_local dump_response_to_file_local read_response_from_file_local

Variables

Type	Visibility	Attributes		Name		Initial
integer,	private		::	dlun	=	6	File unit for debug printing
logical,	public		::	do_smart_update	=	.false.	If true and x/y distributed then use a "smart" update which only updates
logical,	private,	parameter	::	debug	=	.false.	Do we do debug stuff?
logical,	private		::	initialised	=	.false.	Have we initialised yet?
logical,	private		::	reinit	=	.false.	Are we reinitialising?
logical,	public		::	dump_response	=	.false.	Do we dump the response matrix?
logical,	public		::	read_response	=	.false.	Do we read the response matrix from dump?
logical,	public		::	field_local_allreduce	=	.false.	If true use an allreduce to gather field else use reduce+broadcast
logical,	public		::	field_local_allreduce_sub	=	.false.	If true and field_local_allreduce true then do two sub comm all reduces rather than 1
logical,	public		::	field_local_tuneminnrow	=	.false.	Are auto-tuning minnrow
logical,	public		::	field_local_nonblocking_collectives	=	.false.
integer,	private		::	nfield			How many fields
integer,	public		::	minNRow	=	16	Tuning this can influence both init and advance times
type(pc_type),	private,	save	::	pc			This is the parallel control object
type(fieldmat_type),	public,	save	::	fieldmat			This is the top level field matrix object
real,	private,	parameter	::	condition_number_warning_limit	=	sqrt(1.0/epsilon(1.0))	The condition numbers above this limit will trigger a warning

Derived Types

type, public :: fields_local_testing

Public type allowing tests to access otherwise-private routines

Type-Bound Procedures

procedure , public , nopass :: getfield_local Subroutine

type, private :: rowblock_type

This is the lowest level of data object and will hold the actual data involved with the field matrix

Components

Type	Visibility	Attributes		Name
complex,	public,	dimension(:,:), allocatable	::	data	This is the field data
integer,	public		::	row_llim	The row limits this block corresponds to
integer,	public		::	row_ulim	The row limits this block corresponds to
integer,	public		::	col_llim	The col limits this block corresponds to
integer,	public		::	col_ulim	The col limits this block corresponds to
integer,	public		::	nrow	The size of the row block
integer,	public		::	ncol	The size of the row block
complex,	public,	dimension(:), allocatable	::	tmp_sum	Work space for field update
integer,	public		::	ir_ind	Property of rowblock
integer,	public		::	it_ind	Details of parents
integer,	public		::	ik_ind	Details of parents
integer,	public		::	is_ind	Details of parents so cell can find itself
integer,	public		::	ic_ind	Details of parents so cell can find itself

Type-Bound Procedures

procedure , private , :: deallocate => rb_deallocate Subroutine
procedure , private , :: allocate => rb_allocate Subroutine
procedure , private , :: init => rb_init Subroutine
procedure , private , :: debug_print => rb_debug_print Subroutine
procedure , private , :: mv_mult_fun => rb_mv_mult_fun Function
procedure , private , :: mv_mult => rb_mv_mult Subroutine
procedure , private , :: irex_to_ir => rb_irex_to_irloc Function
procedure , private , :: reset => rb_reset Subroutine
procedure , private , :: set_nrow => rb_set_nrow Subroutine

type, private :: cell_type

This is the next level up of data and represents the cell type. A cell has a unique it,ik pair and represents all the fields/field eq on the extended domain for the given it/ik

Components

Type	Visibility	Attributes		Name
type(rowblock_type),	public,	dimension(:), allocatable	::	rb	These are the row blocks, currently one for each field equation
integer,	public		::	nrb	How many row blocks, equal to nfield
integer,	public		::	row_llim	The row limits, always 1 to nfield*nextend
integer,	public		::	row_ulim	The row limits, always 1 to nfield*nextend
integer,	public,	dimension(:), allocatable	::	col_llim	These are the column limits for each fdq
integer,	public,	dimension(:), allocatable	::	col_ulim	These are the column limits for each fdq
integer,	public		::	nrow	The number of rows and columns for each field equation.
integer,	public		::	ncol	The number of rows and columns for each field equation.
integer,	public		::	ncol_tot	The total number of columns (nfdq*ncol)
logical,	public		::	is_local	Does this cell have any data on this proc?
logical,	public		::	is_empty	Have we got any data for this cell on this proc?
logical,	public		::	is_all_local	Is all of this cells data on this proc?
logical,	public		::	ignore_boundary	Do we ignore the boundary point (column) in this cell?
complex,	public,	dimension(:), allocatable	::	tmp_sum	Work space for field update
type(comm_type),	public		::	parent_sub	Sub communicator involving all processors in parent
integer,	public		::	ic_ind	Cell properties
integer,	public		::	it_ind	Cell properties
integer,	public		::	ik_ind	Parent properties
integer,	public		::	is_ind	Parent properties

Type-Bound Procedures

procedure , private , :: deallocate => c_deallocate Subroutine
procedure , private , :: allocate => c_allocate Subroutine
procedure , private , :: init => c_init Subroutine
procedure , private , :: debug_print => c_debug_print Subroutine
procedure , private , :: mv_mult_rb => c_mv_mult_rb Subroutine
procedure , private , :: get_field_update => c_get_field_update Subroutine
procedure , private , :: reset => c_reset Subroutine
procedure , private , :: set_locality => c_set_locality Subroutine
procedure , private , :: has_row => c_has_row Function

type, private :: supercell_type

This is the next level up of data and represents the supercell type. A supercell represents a collection of connected cells

Components

Type	Visibility	Attributes		Name		Initial
type(cell_type),	public,	dimension(:), allocatable	::	cells			These are the cells
integer,	public		::	ncell
integer,	public		::	nextend			Length of the extended domain
integer,	public		::	nrow			The number of rows and columns. Equal to nextend*nfield
integer,	public		::	ncol			The number of rows and columns. Equal to nextend*nfield
integer,	public		::	it_leftmost			It index of leftmost cell
integer,	public		::	head_iproc			The proc id (in sc_sub_all) of the head proc
integer,	public		::	head_iproc_pd			The proc id (in sc_sub_pd) of the head proc
logical,	public		::	is_local			Does this supercell have any data on this proc?
logical,	public		::	is_empty			Have we got any data for this supercell on this proc?
logical,	public		::	is_all_local			Is all of this supercells data on this proc?
complex,	public,	dimension(:), allocatable	::	tmp_sum			Work space for field update
type(comm_type),	public		::	sc_sub_all			Sub communicator involving all processors with this supercell
type(comm_type),	public		::	sc_sub_pd			Sub communicator for all procs with some data but not all of it
type(comm_type),	public		::	parent_sub			Sub communicator involving all processors in parent
integer,	public,	dimension(:), allocatable	::	nb_req_hand			For non-blocking broadcast request handle storage
logical,	public		::	initdone			Have we finished initialising this block?
logical,	public,	dimension(:), allocatable	::	initialised			Have we initialised each point?
logical,	public		::	is_head	=	.false.	Are we the head of this supercell?
integer,	public		::	is_ind			The is_ind value is the index of the supercell in the parent ky_type's supercell array.
integer,	public		::	is_label			The is_label is the supercell_label of this supercell as determined by calculate_supercell_labels from elsewhere.
integer,	public		::	ik_ind			Parent properties
integer,	public		::	collective_request
real,	public		::	condition_number	=	-1	Condition number of the associated response matrix. Only valid on the head.

Type-Bound Procedures

procedure , private , :: deallocate => sc_deallocate Subroutine
procedure , private , :: allocate => sc_allocate Subroutine
procedure , private , :: init => sc_init Subroutine
procedure , private , :: debug_print => sc_debug_print Subroutine
procedure , private , :: get_field_update => sc_get_field_update Subroutine
procedure , private , :: reduce_tmpsum => sc_reduce_tmpsum Subroutine
procedure , private , :: iex_to_dims => sc_iex_to_dims Subroutine
procedure , private , :: iex_to_ifl => sc_iex_to_ifl Subroutine
procedure , private , :: iex_to_ic => sc_iex_to_ic Subroutine
procedure , private , :: iex_to_ig => sc_iex_to_ig Subroutine
procedure , private , :: iex_to_it => sc_iex_to_it Subroutine
procedure , private , :: has_it => sc_has_it Function
procedure , private , :: reset => sc_reset Subroutine
procedure , private , :: set_locality => sc_set_locality Subroutine
procedure , private , :: get_left_it => sc_get_left_it Function
procedure , private , :: store_fq => sc_store_fq Subroutine
procedure , private , :: pull_rows_to_arr => sc_pull_rows_to_arr Subroutine
procedure , private , :: push_arr_to_rows => sc_push_arr_to_rows Subroutine
procedure , private , :: prepare => sc_prepare Subroutine
procedure , private , :: invert => sc_invert Subroutine
procedure , private , :: invert_local => sc_invert_local Subroutine
procedure , private , :: invert_mpi => sc_invert_mpi Subroutine
procedure , private , :: dump => sc_dump Subroutine
procedure , private , :: make_subcom_1 => sc_make_subcom_1 Subroutine
procedure , private , :: make_subcom_2 => sc_make_subcom_2 Subroutine
procedure , private , :: dump_to_file => sc_dump_to_file Subroutine
procedure , private , :: read_from_file => sc_read_from_file Subroutine

type, private :: ky_type

This is the next level up of data and represents the ky type. A ky block consists of a number of supercells

Components

Type	Visibility	Attributes		Name
type(supercell_type),	public,	dimension(:), allocatable	::	supercells	These are the supercells
integer,	public		::	nsupercell
logical,	public		::	is_local	Does this supercell have any data on this proc?
logical,	public		::	is_empty	Have we got any data for this supercell on this proc?
logical,	public		::	is_all_local	Is all of this supercells data on this proc?
logical,	public		::	initdone	Have we finished initialising this block?
type(comm_type),	public		::	ky_sub_all	Sub communicator involving all processors with this ky
type(comm_type),	public		::	parent_sub	Sub communicator involving all processors in parent
integer,	public		::	ik_ind

Type-Bound Procedures

procedure , private , :: deallocate => ky_deallocate Subroutine
procedure , private , :: allocate => ky_allocate Subroutine
procedure , private , :: init => ky_init Subroutine
procedure , private , :: debug_print => ky_debug_print Subroutine
procedure , private , :: get_field_update => ky_get_field_update Subroutine
procedure , private , :: wait_for_nonblocking_collectives => ky_wait_for_nonblocking_collectives Subroutine
procedure , private , :: reset => ky_reset Subroutine
procedure , private , :: set_locality => ky_set_locality Subroutine
procedure , private , :: is_from_it => ky_is_from_it Function
procedure , private , :: store_fq => ky_store_fq Subroutine
procedure , private , :: prepare => ky_prepare Subroutine
procedure , private , :: make_subcom_1 => ky_make_subcom_1 Subroutine
procedure , private , :: make_subcom_2 => ky_make_subcom_2 Subroutine

type, private :: fieldmat_type

This is the top level object, consisting of a collection of ky blocks.

Components

Type	Visibility	Attributes		Name		Initial
type(ky_type),	public,	dimension(:), allocatable	::	kyb			The ky blocks
integer,	public		::	naky			Number of ky blocks
integer,	public		::	ntheta0
integer,	public		::	npts			Total number of theta grid points on all extended domains
integer,	public		::	nbound			Number of ignored boundary points
logical,	public		::	is_local			Does this supercell have any data on this proc?
logical,	public		::	is_empty			Have we got any data for the fieldmat on this proc?
logical,	public		::	is_all_local			Is all of this supercells data on this proc?
type(comm_type),	public		::	fm_sub_all			Sub communicator involving all processors with fieldmat
type(comm_type),	public		::	fm_sub_headsc_p0			Sub communicator involving the supercell heads and proc0
integer,	public		::	prepare_type	=	0	What sort of preparation do we do to the field matrix
integer,	public,	dimension(:,:), allocatable	::	heads			An array that holds the iproc (from comm world) of the supercell head for a given ik,it point
logical,	public		::	no_populate	=	.false.	Advanced usage only, if true then don't populate response
logical,	public		::	no_prepare	=	.false.	Advanced usage only, if true then don't prepare (invert) response

Type-Bound Procedures

procedure , private , :: deallocate => fm_deallocate Subroutine
procedure , private , :: allocate => fm_allocate Subroutine
procedure , private , :: debug_print => fm_debug_print Subroutine
procedure , private , :: get_field_update => fm_get_field_update Subroutine
procedure , private , :: init => fm_init Subroutine
procedure , private , :: reset => fm_reset Subroutine
procedure , private , :: populate => fm_populate Subroutine
procedure , private , :: init_next_field_points => fm_init_next_field_points Subroutine
procedure , private , :: prepare => fm_prepare Subroutine
procedure , private , :: make_subcom_1 => fm_make_subcom_1 Subroutine
procedure , private , :: make_subcom_2 => fm_make_subcom_2 Subroutine
procedure , private , :: calc_sc_heads => fm_calc_sc_heads Subroutine
procedure , private , :: gather_fields => fm_gather_fields Subroutine
procedure , private , :: unpack_to_field => fm_unpack_to_field Subroutine
procedure , private , :: write_debug_data => fm_write_debug_data Subroutine
procedure , private , :: set_is_local => fm_set_is_local Subroutine
procedure , private , :: count_subcom => fm_count_subcom Subroutine
procedure , private , :: getfieldeq_nogath => fm_getfieldeq_nogath Subroutine
procedure , private , :: reduce_an => fm_reduce_an_head_send_broadcast Subroutine
procedure , private , :: check_an => fm_check_an Subroutine
procedure , private , :: getfieldeq1_nogath => fm_getfieldeq1_nogath Subroutine
procedure , private , :: update_fields => fm_update_fields Subroutine
procedure , private , :: update_fields_newstep => fm_update_fields_newstep Subroutine
procedure , private , :: get_condition_numbers => fm_get_condition_numbers Subroutine

type, private :: pc_type

This is the type which controls/organises the parallel data decomposition etc.

Components

Type	Visibility	Attributes		Name		Initial
integer,	public,	dimension(:,:), allocatable	::	is_local			Is the given it,ik on this proc?
type(comm_type),	public,	dimension(:,:), allocatable	::	itik_subcom			Cell level sub-communicators
integer,	public,	dimension(:,:), allocatable	::	nproc_per_cell			How many procs have this cell
integer,	public,	dimension(:,:), allocatable	::	nresp_per_cell			How many rows are we personally responsible for in each cell
integer,	public,	dimension(:,:), allocatable	::	navail_per_cell			How many rows could we be responsible for in each cell
integer,	public		::	nresp_tot			Total number of rows available/potentially available
integer,	public		::	navail_tot			Total number of rows available/potentially available
logical,	public		::	force_local_invert	=	.false.	If true then we force local inversion
logical,	public		::	has_to_gather	=	.true.	If true we have to gather when calling getfieldeq, determined by decomp routine
integer,	public		::	decomp_type	=	3	This sets what type of decomposition is done

Type-Bound Procedures

procedure , private , :: deallocate => pc_deallocate Subroutine
procedure , private , :: allocate => pc_allocate Subroutine
procedure , private , :: debug_print => pc_debug_print Subroutine
procedure , private , :: current_nresp => pc_current_nresp Function
procedure , private , :: find_locality => pc_find_locality Subroutine
procedure , private , :: count_avail => pc_count_avail Subroutine
procedure , private , :: has_ik => pc_has_ik Function
procedure , private , :: has_it => pc_has_it Function
procedure , private , :: make_decomp => pc_make_decomp Subroutine
procedure , private , :: init => pc_init Subroutine
procedure , private , :: reset => pc_reset Subroutine
procedure , private , :: decomp_all_serial_local => pc_decomp_all_serial_local Subroutine
procedure , private , :: decomp_own_serial_local => pc_decomp_own_serial_local Subroutine
procedure , private , :: decomp_owncells_serial_local => pc_decomp_owncells_serial_local Subroutine
procedure , private , :: decomp_owncells_simplempi => pc_decomp_owncells_simplempi Subroutine

Functions

private function rb_mv_mult_fun(self, vect)

Matrix vector multiplication

Arguments

Type	Intent	Optional	Attributes		Name
class(rowblock_type),	intent(in)			::	self
complex,	intent(in),		dimension(self%ncol)	::	vect

Return Value complex, dimension(self%nrow)

private function rb_irex_to_irloc(self, irex)

Convert extended row index to local row index

Arguments

Type	Intent	Optional	Attributes		Name
class(rowblock_type),	intent(in)			::	self
integer,	intent(in)			::	irex

Return Value integer

private function c_has_row(self, irow)

Test if a given row belongs to the current cell

Arguments

Type	Intent	Optional	Attributes		Name
class(cell_type),	intent(in)			::	self
integer,	intent(in)			::	irow

Return Value logical

private function sc_has_it(self, it)

Is the passed it a member of this supercell

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(in)			::	self
integer,	intent(in)			::	it

Return Value logical

private function sc_get_left_it(self, it)

Given an it value get the it of the left connected cell

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(in)			::	self
integer,	intent(in)			::	it

Return Value integer

private function ky_is_from_it(self, it)

Given it say what the supercell id is

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self
integer,	intent(in)			::	it

Return Value integer

private function pc_current_nresp(self)

Return the current number of rows we're responsible for

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(in)			::	self

Return Value integer

private function pc_has_ik(self, ik)

Determine if we have any cells with passed ik.

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(in)			::	self
integer,	intent(in)			::	ik

Return Value logical

private function pc_has_it(self, it)

Determine if we have any cells with passed it.

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(in)			::	self
integer,	intent(in)			::	it

Return Value logical

public function fields_local_functional()

Returns true if GS2 was built in such a way as to allow this module to work. Currently does not work with PGI compilers. See online discussions.

Arguments

None

Return Value logical

private function l2norm(matrix)

Calculate the L2/Frobenius/Euclidiean-norm of a 2D complex array

Arguments

Type	Intent	Optional	Attributes		Name
complex,	intent(in),		dimension(:, :)	::	matrix	The matrix to calculate the norm of

Return Value real

Subroutines

private subroutine rb_allocate(self)

Allocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(rowblock_type),	intent(inout)			::	self

private subroutine rb_deallocate(self)

Deallocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(rowblock_type),	intent(inout)			::	self

private subroutine rb_init(self, ifq, ic, it, is, ik, ncol, nextend)

Initialise the members of the rowblock instance

Arguments

Type	Intent		Name
class(rowblock_type),	intent(inout)	::	self
integer,	intent(in)	::	ifq
integer,	intent(in)	::	ic
integer,	intent(in)	::	it
integer,	intent(in)	::	is
integer,	intent(in)	::	ik
integer,	intent(in)	::	ncol
integer,	intent(in)	::	nextend	The length of the parent supercell's real space domain. Represents the spacing between the start of different field/field equations as we store data as the full real space domain for each field, e.g. [fieldeq(:), fieldeqa(:), fieldeqp(:)], rather than say interleaved, e.g. [[fieldeq(1), fieldeqa(1), fieldeqp(1)], [fieldeq(2), fieldeqa(2), fieldeqp(2)],...]

private subroutine rb_debug_print(self)

Debug printing

Arguments

Type	Intent	Optional	Attributes		Name
class(rowblock_type),	intent(in)			::	self

private subroutine rb_mv_mult(self, vect)

Matrix vector multiplication stored in tmp_sum

Arguments

Type	Intent	Optional	Attributes		Name
class(rowblock_type),	intent(inout)			::	self
complex,	intent(in),		dimension(self%ncol)	::	vect

private subroutine rb_set_nrow(self)

Set the nrow variables

Arguments

Type	Intent	Optional	Attributes		Name
class(rowblock_type),	intent(inout)			::	self

private subroutine rb_reset(self)

A routine to reset the object

Arguments

Type	Intent	Optional	Attributes		Name
class(rowblock_type),	intent(inout)			::	self

private subroutine c_allocate(self)

Allocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(cell_type),	intent(inout)			::	self

private subroutine c_deallocate(self)

Deallocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(cell_type),	intent(inout)			::	self

private subroutine c_init(self, ic, it, is, ik, nfield, nrow, nextend, ignore_boundary)

Initialise the members of the cell instance

Arguments

Type	Intent		Name
class(cell_type),	intent(inout)	::	self
integer,	intent(in)	::	ic
integer,	intent(in)	::	it
integer,	intent(in)	::	is
integer,	intent(in)	::	ik
integer,	intent(in)	::	nfield
integer,	intent(in)	::	nrow
integer,	intent(in)	::	nextend
logical,	intent(in)	::	ignore_boundary

private subroutine c_debug_print(self)

Debug printing

Arguments

Type	Intent	Optional	Attributes		Name
class(cell_type),	intent(in)			::	self

private subroutine c_mv_mult_rb(self, vect, ifq)

Do matrix vector multiplication at rowblock level

Arguments

Type	Intent	Attributes		Name
class(cell_type),	intent(inout)		::	self
complex,	intent(in),	dimension(self%ncol)	::	vect
integer,	intent(in)		::	ifq

private subroutine c_get_field_update(self, fq, fqa, fqp)

Get the field update for this cells data Note still need to reduce across other cells in this supercell.

Arguments

Type	Intent	Attributes		Name
class(cell_type),	intent(inout)		::	self
complex,	intent(in),	dimension(self%ncol)	::	fq
complex,	intent(in),	dimension(self%ncol)	::	fqa
complex,	intent(in),	dimension(self%ncol)	::	fqp

private subroutine c_reset(self)

A routine to reset the object

Arguments

Type	Intent	Optional	Attributes		Name
class(cell_type),	intent(inout)			::	self

private subroutine c_set_locality(self)

Set the locality of each object

Arguments

Type	Intent	Optional	Attributes		Name
class(cell_type),	intent(inout)			::	self

private subroutine sc_allocate(self)

Allocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_deallocate(self)

Deallocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_init(self, is, itmin, ik, nfield, nbound)

Initialise the supercell instance by setting and calculating some basic properties. Does not deal with allocating all storage etc.

Arguments

Type	Intent		Name
class(supercell_type),	intent(inout)	::	self
integer,	intent(in)	::	is
integer,	intent(in)	::	itmin
integer,	intent(in)	::	ik
integer,	intent(in)	::	nfield
integer,	intent(inout)	::	nbound

private subroutine sc_debug_print(self)

Debug printing

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(in)			::	self

private subroutine sc_get_field_update(self, fq, fqa, fqp)

Get the field update DD>TAGGED

Arguments

Type	Intent	Attributes		Name
class(supercell_type),	intent(inout)		::	self
complex,	intent(in),	dimension(:,:)	::	fq
complex,	intent(in),	dimension(:,:)	::	fqa
complex,	intent(in),	dimension(:,:)	::	fqp

private subroutine sc_reduce_tmpsum(self)

Reduce the field update across cells to give the final answer DD>TAGGED: As we currently have to do fm_gather_fields on every time step we only need

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_iex_to_dims(self, iex, ig, ic, it, ifl)

Convert the extended domain index to ig, it and ifl

Arguments

Type	Intent		Name
class(supercell_type),	intent(in)	::	self
integer,	intent(in)	::	iex
integer,	intent(out)	::	ig
integer,	intent(out)	::	ic
integer,	intent(out)	::	it
integer,	intent(out)	::	ifl

private subroutine sc_iex_to_ifl(self, iex, ifl)

Convert the extended domain index to ifl

Arguments

Type	Intent		Name
class(supercell_type),	intent(in)	::	self
integer,	intent(in)	::	iex
integer,	intent(out)	::	ifl

private subroutine sc_iex_to_ic(self, iex, ic)

Convert the extended domain index to ic

Arguments

Type	Intent		Name
class(supercell_type),	intent(in)	::	self
integer,	intent(in)	::	iex
integer,	intent(out)	::	ic

private subroutine sc_iex_to_it(self, iex, it)

Convert the extended domain index to it

Arguments

Type	Intent		Name
class(supercell_type),	intent(in)	::	self
integer,	intent(in)	::	iex
integer,	intent(out)	::	it

private subroutine sc_iex_to_ig(self, iex, ig)

Convert the extended domain index to ig

Arguments

Type	Intent		Name
class(supercell_type),	intent(in)	::	self
integer,	intent(in)	::	iex
integer,	intent(out)	::	ig

private subroutine sc_reset(self)

A routine to reset the object

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_set_locality(self)

Set the locality of each object

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_dump(self, prefix)

Debug routine to dump the current supercell

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self
character(len=*),	intent(in),	optional		::	prefix

private subroutine sc_dump_to_file(self, suffix)

Routine to write the response matrix for this supercell to netcdf file.

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self	The instance of the supercell class
character(len=*),	intent(in),	optional		::	suffix	If passed then use as part of file suffix

private subroutine sc_read_from_file(self, could_read, suffix)

Routine to read the response matrix for this supercell from netcdf file.

Arguments

Type	Intent	Optional		Name
class(supercell_type),	intent(inout)		::	self	The instance of the supercell class
logical,	intent(out)		::	could_read	Flag to indicate if the file was successfully read
character(len=*),	intent(in),	optional	::	suffix	If passed then use as part of file suffix

private subroutine sc_store_fq(self, fq, fqa, fqp, ifl_in, it_in, ig_in)

Store the field equations at row level

Arguments

Type	Intent	Attributes		Name
class(supercell_type),	intent(inout)		::	self
complex,	intent(in),	dimension(:, :)	::	fq
complex,	intent(in),	dimension(:, :)	::	fqa
complex,	intent(in),	dimension(:, :)	::	fqp
integer,	intent(in)		::	ifl_in
integer,	intent(in)		::	it_in
integer,	intent(in)		::	ig_in

private subroutine sc_prepare(self, prepare_type)

Prepare the field matrix for calculating field updates

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self
integer,	intent(in)			::	prepare_type

private subroutine sc_invert(self)

A routine to invert the field matrix

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_invert_local(self)

A routine to invert the field matrix locally

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_invert_mpi(self)

A routine to invert the field matrix using mpi

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_pull_rows_to_arr(self, arr)

A routine to collect all the row level data and store in passed array Gather the row blocks up for this cell to fill an array DD>FOR NOW USE ALL_REDUCE AS EASIER, BUT SHOULD BE ABLE

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(in)			::	self
complex,	intent(out),		dimension(:,:)	::	arr

private subroutine sc_push_arr_to_rows(self, arr)

A routine to distribute an array to appropriate row blocks

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self
complex,	intent(in),		dimension(self%nrow,self%ncol)	::	arr

private subroutine sc_make_subcom_1(self)

Create primary (top level) sub communicators

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine sc_make_subcom_2(self)

Create the secondary (intraobject) subcommunicators

Arguments

Type	Intent	Optional	Attributes		Name
class(supercell_type),	intent(inout)			::	self

private subroutine ky_allocate(self)

Allocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self

private subroutine ky_deallocate(self)

Deallocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self

private subroutine ky_init(self, ik, itmins, nsupercell, nfield, nbound)

Setup properties of ky_type instance

Arguments

Type	Intent	Attributes		Name
class(ky_type),	intent(inout)		::	self
integer,	intent(in)		::	ik
integer,	intent(in),	dimension(:)	::	itmins
integer,	intent(in)		::	nsupercell
integer,	intent(in)		::	nfield
integer,	intent(inout)		::	nbound

private subroutine ky_debug_print(self)

Debug printing

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(in)			::	self

private subroutine ky_get_field_update(self, fq, fqa, fqp)

Get the field update for this ik

Arguments

Type	Intent	Attributes		Name
class(ky_type),	intent(inout)		::	self
complex,	intent(in),	dimension(:,:)	::	fq
complex,	intent(in),	dimension(:,:)	::	fqa
complex,	intent(in),	dimension(:,:)	::	fqp

private subroutine ky_wait_for_nonblocking_collectives(self)

Wait on the nonblocking collective communications if they have been used.

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self

private subroutine ky_store_fq(self, fq, fqa, fqp, ifl_in, it_in, ig_in)

Store the field equation at row level

Arguments

Type	Intent	Attributes		Name
class(ky_type),	intent(inout)		::	self
complex,	intent(in),	dimension(:, :)	::	fq
complex,	intent(in),	dimension(:, :)	::	fqa
complex,	intent(in),	dimension(:, :)	::	fqp
integer,	intent(in)		::	ifl_in
integer,	intent(in)		::	it_in
integer,	intent(in)		::	ig_in

private subroutine ky_reset(self)

A routine to reset the object

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self

private subroutine ky_set_locality(self)

Set the locality of each object

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self

private subroutine ky_prepare(self, prepare_type)

Prepare the field matrix for calculating field updates

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self
integer,	intent(in)			::	prepare_type

private subroutine ky_make_subcom_1(self)

Create the primary subcommunicators

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self

private subroutine ky_make_subcom_2(self)

Create the secondary subcommunicators

Arguments

Type	Intent	Optional	Attributes		Name
class(ky_type),	intent(inout)			::	self

private subroutine tuneMinNRow()

Arguments

None

private subroutine fm_allocate(self)

Allocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_deallocate(self)

Deallocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_debug_print(self)

Debug printing

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(in)			::	self

private subroutine fm_set_is_local(self)

Just work out the locality (also sets is_empty etc. but is not intended for this)

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_init(self)

Initialise the field objects

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_populate(self)

Find the response of g to delta-fn field perturbations and store at the row level

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_init_next_field_points(self, field, pts_remain, kwork_filter, ifl)

Initialise the next set of delta functions, find the response and store in the appropriate rows. DD>Could improve performance by using a "smart" routine which only operates on local/not empty data

Arguments

Type	Intent	Attributes		Name
class(fieldmat_type),	intent(inout)		::	self
complex,	intent(inout),	dimension(-ntgrid:ntgrid,ntheta0,naky)	::	field
integer,	intent(inout)		::	pts_remain
logical,	intent(inout),	dimension(ntheta0, naky)	::	kwork_filter
integer,	intent(in)		::	ifl

private subroutine fm_prepare(self)

Prepare the field matrix for calculating field updates

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_get_field_update(self, phi, apar, bpar)

A routine to calculate the update to the fields DD>Could improve performance by using a "smart" routine which only operates on local/not empty data

Arguments

Type	Intent	Attributes		Name
class(fieldmat_type),	intent(inout)		::	self
complex,	intent(inout),	dimension(:,:,:)	::	phi
complex,	intent(inout),	dimension(:,:,:)	::	apar
complex,	intent(inout),	dimension(:,:,:)	::	bpar

private subroutine fm_unpack_to_field(self, ph, ap, bp)

A routine to unpack the supercell tmp_sum vectors to full field arrays

Arguments

Type	Intent	Attributes		Name
class(fieldmat_type),	intent(inout)		::	self
complex,	intent(inout),	dimension(-ntgrid:,:,:)	::	ph
complex,	intent(inout),	dimension(-ntgrid:,:,:)	::	ap
complex,	intent(inout),	dimension(-ntgrid:,:,:)	::	bp

private subroutine fm_reset(self)

A routine to reset the object

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_count_subcom(self)

Count how many subcommunicators will be created

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_make_subcom_1(self)

Create all the necessary subcommunicators

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_make_subcom_2(self)

Create the secondary subcommunicators

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_calc_sc_heads(self)

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self

private subroutine fm_gather_fields(self, ph, ap, bp, to_all_in, do_allreduce_in)

Gather all the fields to proc0/all for diagnostics etc. DD>TAGGED:Worth looking at improving this bad memory pattern

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(inout)			::	self
complex,	intent(inout),		dimension(:,:,:)	::	ph
complex,	intent(inout),		dimension(:,:,:)	::	ap
complex,	intent(inout),		dimension(:,:,:)	::	bp
logical,	intent(in),	optional		::	to_all_in
logical,	intent(in),	optional		::	do_allreduce_in

private subroutine fm_update_fields(self, phi, apar, bpar)

Update the fields using calculated update

Arguments

Type	Intent	Attributes		Name
class(fieldmat_type),	intent(in)		::	self
complex,	intent(inout),	dimension(:,:,:)	::	phi
complex,	intent(inout),	dimension(:,:,:)	::	apar
complex,	intent(inout),	dimension(:,:,:)	::	bpar

private subroutine fm_update_fields_newstep(self)

Update the fields using the new fields

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(in)			::	self

private subroutine fm_get_condition_numbers(self)

Fetch all condition numbers to proc0 and report on worrying values as required.

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(in)			::	self

private subroutine fm_write_debug_data(self)

Write some debug data to file

Arguments

Type	Intent	Optional	Attributes		Name
class(fieldmat_type),	intent(in)			::	self

private subroutine fm_getfieldeq_nogath(self, phi, apar, bpar, fieldeq, fieldeqa, fieldeqp)

Arguments

Type	Intent	Attributes		Name
class(fieldmat_type),	intent(in)		::	self
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	phi
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	apar
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	bpar
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	fieldeq
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	fieldeqa
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	fieldeqp

private subroutine fm_reduce_an_head_send_broadcast(self, antot, antota, antotp)

Arguments

Type	Intent	Attributes		Name
class(fieldmat_type),	intent(in)		::	self
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	antot
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	antota
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	antotp

private subroutine fm_check_an(self, antot, tempantot, antota, tempantota, antotp, tempantotp)

Arguments

Type	Intent	Attributes		Name
class(fieldmat_type),	intent(in)		::	self
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	antot
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	tempantot
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	antota
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	tempantota
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	antotp
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	tempantotp

private subroutine fm_getfieldeq1_nogath(self, phi, apar, bpar, antot, antota, antotp, fieldeq, fieldeqa, fieldeqp)

Arguments

Type	Intent	Attributes		Name
class(fieldmat_type),	intent(in)		::	self
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	phi
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	apar
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	bpar
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	antot
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	antota
complex,	intent(in),	dimension (-ntgrid:,:,:)	::	antotp
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	fieldeq
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	fieldeqa
complex,	intent(inout),	dimension (-ntgrid:,:,:)	::	fieldeqp

private subroutine pc_allocate(self)

Allocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_deallocate(self)

Deallocate storage space

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_debug_print(self)

Debug printing

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(in)			::	self

private subroutine pc_find_locality(self)

Work out which cells are local

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_count_avail(self)

Work out how many rows are available in each cell

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_make_decomp(self)

A wrapper routine to do the decomposition

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_reset(self)

A routine to reset the pc object

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_init(self)

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_decomp_all_serial_local(self)

All serial and local || Decomp_type=0

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_decomp_own_serial_local(self)

Serial and local but only for the supercells || Decomp_type=1 for which we have some data

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_decomp_owncells_serial_local(self)

Serial and local but only for the cells || Decomp_type=2 for which we have some data

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

private subroutine pc_decomp_owncells_simplempi(self)

Cells are decomposed based on proc || Decomp_type=3

Arguments

Type	Intent	Optional	Attributes		Name
class(pc_type),	intent(inout)			::	self

public subroutine init_fields_local()

Routine to initialise

Arguments

None

private subroutine init_fields_matrixlocal(tuning_in)

Routine use to initialise field matrix data !!!!!!!! NOTE: All of the above should be a one off setup cost unless something fundamental changes with the parallel layout (a change in nproc/layout etc.) The remaining two tasks (populate+prepare) must be repeated each time the relevant physics and numerical parameters change. In practice this just means when the time step changes. !!!!!!!!

Arguments

Type	Intent	Optional	Attributes		Name
logical,		optional		::	tuning_in

public subroutine reset_fields_local()

Reset the fields_local module

Arguments

None

public subroutine finish_fields_local()

Finish the fields_local module

Arguments

None

private subroutine getfield_local(phi, apar, bpar, do_gather_in, do_update_in)

Calculate the update to the fields

Arguments

Type	Intent	Optional	Attributes		Name
complex,	intent(out),		dimension(:,:,:)	::	phi
complex,	intent(out),		dimension(:,:,:)	::	apar
complex,	intent(out),		dimension(:,:,:)	::	bpar
logical,	intent(in),	optional		::	do_gather_in
logical,	intent(in),	optional		::	do_update_in

public subroutine init_allfields_local()

Initialise the fields from the initial g, just uses the fields_implicit routine

Arguments

None

public subroutine advance_local(istep, remove_zonal_flows_switch)

This routine advances the solution by one full time step DD>TAGGED: Should we only this is fapar>0 as well?

Arguments

Type	Intent	Optional	Attributes		Name
integer,	intent(in)			::	istep
logical,	intent(in)			::	remove_zonal_flows_switch

public subroutine dump_response_to_file_local(suffix)

Routine to dump the current response matrix data to file. One file per connected domain. Each written by the head of the supercell.

Arguments

Type	Intent	Optional	Attributes		Name
character(len=*),	intent(in),	optional		::	suffix	If passed then use as part of file suffix

public subroutine read_response_from_file_local(could_read, suffix)

Routine to read the current response matrix data from file dump. One file per connected domain. Each written by the head of the supercell. NOTE: Have to have setup communicators etc.

Arguments

Type	Intent	Optional	Attributes		Name
logical,	intent(out)			::	could_read	Flag to indicate if all files successfully read
character(len=*),	intent(in),	optional		::	suffix	If passed then use as part of file suffix

fields_local Module

Contents

Uses

Contents

Variables

Derived Types

type, public :: fields_local_testing

Type-Bound Procedures

type, private :: rowblock_type

Components

Type-Bound Procedures

type, private :: cell_type

Components

Type-Bound Procedures

type, private :: supercell_type

Components

Type-Bound Procedures

type, private :: ky_type

Components

Type-Bound Procedures

type, private :: fieldmat_type

Components

Type-Bound Procedures

type, private :: pc_type

Components

Type-Bound Procedures

Functions

private function rb_mv_mult_fun(self, vect)

Arguments

Return Value complex, dimension(self%nrow)

private function rb_irex_to_irloc(self, irex)

Arguments

Return Value integer

private function c_has_row(self, irow)

Arguments

Return Value logical

private function sc_has_it(self, it)

Arguments

Return Value logical

private function sc_get_left_it(self, it)

Arguments

Return Value integer

private function ky_is_from_it(self, it)

Arguments

Return Value integer

private function pc_current_nresp(self)

Arguments

Return Value integer

private function pc_has_ik(self, ik)

Arguments

Return Value logical

private function pc_has_it(self, it)

Arguments

Return Value logical

public function fields_local_functional()

Arguments

Return Value logical

private function l2norm(matrix)

Arguments

Return Value real

Subroutines

private subroutine rb_allocate(self)

Arguments

private subroutine rb_deallocate(self)

Arguments

private subroutine rb_init(self, ifq, ic, it, is, ik, ncol, nextend)

Arguments

private subroutine rb_debug_print(self)

Arguments

private subroutine rb_mv_mult(self, vect)

Arguments

private subroutine rb_set_nrow(self)

Arguments

private subroutine rb_reset(self)

Arguments

private subroutine c_allocate(self)

Arguments

private subroutine c_deallocate(self)

Arguments

private subroutine c_init(self, ic, it, is, ik, nfield, nrow, nextend, ignore_boundary)