fields_local Module

This module implements the same implicit algorithm as fields_implicit. However, it is an almost complete rewrite of the implementation which can give large gains in the speed of calculating the response matrix, as well as significant gains to the advance steps


Uses


Contents

rb_allocate rb_deallocate rb_init rb_debug_print rb_mv_mult rb_set_nrow rb_reset c_allocate c_deallocate c_init c_debug_print c_mv_mult_rb c_get_field_update c_reset c_set_locality sc_allocate sc_deallocate sc_init sc_debug_print sc_get_field_update sc_reduce_tmpsum sc_iex_to_dims sc_iex_to_ifl sc_iex_to_ic sc_iex_to_it sc_iex_to_ig sc_reset sc_set_locality sc_dump sc_dump_to_file sc_read_from_file sc_store_fq sc_prepare sc_invert sc_invert_local sc_invert_mpi sc_pull_rows_to_arr sc_push_arr_to_rows sc_make_subcom_1 sc_make_subcom_2 ky_allocate ky_deallocate ky_init ky_debug_print ky_get_field_update ky_wait_for_nonblocking_collectives ky_store_fq ky_reset ky_set_locality ky_prepare ky_make_subcom_1 ky_make_subcom_2 tuneMinNRow fm_allocate fm_deallocate fm_debug_print fm_set_is_local fm_init fm_populate fm_init_next_field_points fm_prepare fm_get_field_update fm_unpack_to_field fm_reset fm_count_subcom fm_make_subcom_1 fm_make_subcom_2 fm_calc_sc_heads fm_gather_fields fm_update_fields fm_update_fields_newstep fm_get_condition_numbers fm_write_debug_data fm_getfieldeq_nogath fm_reduce_an_head_send_broadcast fm_check_an fm_getfieldeq1_nogath pc_allocate pc_deallocate pc_debug_print pc_find_locality pc_count_avail pc_make_decomp pc_reset pc_init pc_decomp_all_serial_local pc_decomp_own_serial_local pc_decomp_owncells_serial_local pc_decomp_owncells_simplempi init_fields_local init_fields_matrixlocal reset_fields_local finish_fields_local getfield_local init_allfields_local advance_local dump_response_to_file_local read_response_from_file_local

Variables

Type Visibility Attributes Name Initial
integer, private :: dlun = 6

File unit for debug printing

logical, public :: do_smart_update = .false.

If true and x/y distributed then use a "smart" update which only updates

logical, private, parameter :: debug = .false.

Do we do debug stuff?

logical, private :: initialised = .false.

Have we initialised yet?

logical, private :: reinit = .false.

Are we reinitialising?

logical, public :: dump_response = .false.

Do we dump the response matrix?

logical, public :: read_response = .false.

Do we read the response matrix from dump?

logical, public :: field_local_allreduce = .false.

If true use an allreduce to gather field else use reduce+broadcast

logical, public :: field_local_allreduce_sub = .false.

If true and field_local_allreduce true then do two sub comm all reduces rather than 1

logical, public :: field_local_tuneminnrow = .false.

Are auto-tuning minnrow

logical, public :: field_local_nonblocking_collectives = .false.
integer, private :: nfield

How many fields

integer, public :: minNRow = 16

Tuning this can influence both init and advance times

type(pc_type), private, save :: pc

This is the parallel control object

type(fieldmat_type), public, save :: fieldmat

This is the top level field matrix object

real, private, parameter :: condition_number_warning_limit = sqrt(1.0/epsilon(1.0))

The condition numbers above this limit will trigger a warning


Derived Types

type, public ::  fields_local_testing

Public type allowing tests to access otherwise-private routines

Type-Bound Procedures

procedure , public , nopass :: getfield_local Subroutine

type, private ::  rowblock_type

This is the lowest level of data object and will hold the actual data involved with the field matrix

Components

Type Visibility Attributes Name Initial
complex, public, dimension(:,:), allocatable :: data

This is the field data

integer, public :: row_llim

The row limits this block corresponds to

integer, public :: row_ulim

The row limits this block corresponds to

integer, public :: col_llim

The col limits this block corresponds to

integer, public :: col_ulim

The col limits this block corresponds to

integer, public :: nrow

The size of the row block

integer, public :: ncol

The size of the row block

complex, public, dimension(:), allocatable :: tmp_sum

Work space for field update

integer, public :: ir_ind

Property of rowblock

integer, public :: it_ind

Details of parents

integer, public :: ik_ind

Details of parents

integer, public :: is_ind

Details of parents so cell can find itself

integer, public :: ic_ind

Details of parents so cell can find itself

Type-Bound Procedures

procedure , private , :: deallocate => rb_deallocate Subroutine
procedure , private , :: allocate => rb_allocate Subroutine
procedure , private , :: init => rb_init Subroutine
procedure , private , :: debug_print => rb_debug_print Subroutine
procedure , private , :: mv_mult_fun => rb_mv_mult_fun Function
procedure , private , :: mv_mult => rb_mv_mult Subroutine
procedure , private , :: irex_to_ir => rb_irex_to_irloc Function
procedure , private , :: reset => rb_reset Subroutine
procedure , private , :: set_nrow => rb_set_nrow Subroutine

type, private ::  cell_type

This is the next level up of data and represents the cell type. A cell has a unique it,ik pair and represents all the fields/field eq on the extended domain for the given it/ik

Components

Type Visibility Attributes Name Initial
type(rowblock_type), public, dimension(:), allocatable :: rb

These are the row blocks, currently one for each field equation

integer, public :: nrb

How many row blocks, equal to nfield

integer, public :: row_llim

The row limits, always 1 to nfield*nextend

integer, public :: row_ulim

The row limits, always 1 to nfield*nextend

integer, public, dimension(:), allocatable :: col_llim

These are the column limits for each fdq

integer, public, dimension(:), allocatable :: col_ulim

These are the column limits for each fdq

integer, public :: nrow

The number of rows and columns for each field equation.

integer, public :: ncol

The number of rows and columns for each field equation.

integer, public :: ncol_tot

The total number of columns (nfdq*ncol)

logical, public :: is_local

Does this cell have any data on this proc?

logical, public :: is_empty

Have we got any data for this cell on this proc?

logical, public :: is_all_local

Is all of this cells data on this proc?

logical, public :: ignore_boundary

Do we ignore the boundary point (column) in this cell?

complex, public, dimension(:), allocatable :: tmp_sum

Work space for field update

type(comm_type), public :: parent_sub

Sub communicator involving all processors in parent

integer, public :: ic_ind

Cell properties

integer, public :: it_ind

Cell properties

integer, public :: ik_ind

Parent properties

integer, public :: is_ind

Parent properties

Type-Bound Procedures

procedure , private , :: deallocate => c_deallocate Subroutine
procedure , private , :: allocate => c_allocate Subroutine
procedure , private , :: init => c_init Subroutine
procedure , private , :: debug_print => c_debug_print Subroutine
procedure , private , :: mv_mult_rb => c_mv_mult_rb Subroutine
procedure , private , :: get_field_update => c_get_field_update Subroutine
procedure , private , :: reset => c_reset Subroutine
procedure , private , :: set_locality => c_set_locality Subroutine
procedure , private , :: has_row => c_has_row Function

type, private ::  supercell_type

This is the next level up of data and represents the supercell type. A supercell represents a collection of connected cells

Components

Type Visibility Attributes Name Initial
type(cell_type), public, dimension(:), allocatable :: cells

These are the cells

integer, public :: ncell
integer, public :: nextend

Length of the extended domain

integer, public :: nrow

The number of rows and columns. Equal to nextend*nfield

integer, public :: ncol

The number of rows and columns. Equal to nextend*nfield

integer, public :: it_leftmost

It index of leftmost cell

integer, public :: head_iproc

The proc id (in sc_sub_all) of the head proc

integer, public :: head_iproc_pd

The proc id (in sc_sub_pd) of the head proc

logical, public :: is_local

Does this supercell have any data on this proc?

logical, public :: is_empty

Have we got any data for this supercell on this proc?

logical, public :: is_all_local

Is all of this supercells data on this proc?

complex, public, dimension(:), allocatable :: tmp_sum

Work space for field update

type(comm_type), public :: sc_sub_all

Sub communicator involving all processors with this supercell

type(comm_type), public :: sc_sub_pd

Sub communicator for all procs with some data but not all of it

type(comm_type), public :: parent_sub

Sub communicator involving all processors in parent

integer, public, dimension(:), allocatable :: nb_req_hand

For non-blocking broadcast request handle storage

logical, public :: initdone

Have we finished initialising this block?

logical, public, dimension(:), allocatable :: initialised

Have we initialised each point?

logical, public :: is_head = .false.

Are we the head of this supercell?

integer, public :: is_ind

The is_ind value is the index of the supercell in the parent ky_type's supercell array.

integer, public :: is_label

The is_label is the supercell_label of this supercell as determined by calculate_supercell_labels from elsewhere.

integer, public :: ik_ind

Parent properties

integer, public :: collective_request
real, public :: condition_number = -1

Condition number of the associated response matrix. Only valid on the head.

Type-Bound Procedures

procedure , private , :: deallocate => sc_deallocate Subroutine
procedure , private , :: allocate => sc_allocate Subroutine
procedure , private , :: init => sc_init Subroutine
procedure , private , :: debug_print => sc_debug_print Subroutine
procedure , private , :: get_field_update => sc_get_field_update Subroutine
procedure , private , :: reduce_tmpsum => sc_reduce_tmpsum Subroutine
procedure , private , :: iex_to_dims => sc_iex_to_dims Subroutine
procedure , private , :: iex_to_ifl => sc_iex_to_ifl Subroutine
procedure , private , :: iex_to_ic => sc_iex_to_ic Subroutine
procedure , private , :: iex_to_ig => sc_iex_to_ig Subroutine
procedure , private , :: iex_to_it => sc_iex_to_it Subroutine
procedure , private , :: has_it => sc_has_it Function
procedure , private , :: reset => sc_reset Subroutine
procedure , private , :: set_locality => sc_set_locality Subroutine
procedure , private , :: get_left_it => sc_get_left_it Function
procedure , private , :: store_fq => sc_store_fq Subroutine
procedure , private , :: pull_rows_to_arr => sc_pull_rows_to_arr Subroutine
procedure , private , :: push_arr_to_rows => sc_push_arr_to_rows Subroutine
procedure , private , :: prepare => sc_prepare Subroutine
procedure , private , :: invert => sc_invert Subroutine
procedure , private , :: invert_local => sc_invert_local Subroutine
procedure , private , :: invert_mpi => sc_invert_mpi Subroutine
procedure , private , :: dump => sc_dump Subroutine
procedure , private , :: make_subcom_1 => sc_make_subcom_1 Subroutine
procedure , private , :: make_subcom_2 => sc_make_subcom_2 Subroutine
procedure , private , :: dump_to_file => sc_dump_to_file Subroutine
procedure , private , :: read_from_file => sc_read_from_file Subroutine

type, private ::  ky_type

This is the next level up of data and represents the ky type. A ky block consists of a number of supercells

Components

Type Visibility Attributes Name Initial
type(supercell_type), public, dimension(:), allocatable :: supercells

These are the supercells

integer, public :: nsupercell
logical, public :: is_local

Does this supercell have any data on this proc?

logical, public :: is_empty

Have we got any data for this supercell on this proc?

logical, public :: is_all_local

Is all of this supercells data on this proc?

logical, public :: initdone

Have we finished initialising this block?

type(comm_type), public :: ky_sub_all

Sub communicator involving all processors with this ky

type(comm_type), public :: parent_sub

Sub communicator involving all processors in parent

integer, public :: ik_ind

Type-Bound Procedures

procedure , private , :: deallocate => ky_deallocate Subroutine
procedure , private , :: allocate => ky_allocate Subroutine
procedure , private , :: init => ky_init Subroutine
procedure , private , :: debug_print => ky_debug_print Subroutine
procedure , private , :: get_field_update => ky_get_field_update Subroutine
procedure , private , :: wait_for_nonblocking_collectives => ky_wait_for_nonblocking_collectives Subroutine
procedure , private , :: reset => ky_reset Subroutine
procedure , private , :: set_locality => ky_set_locality Subroutine
procedure , private , :: is_from_it => ky_is_from_it Function
procedure , private , :: store_fq => ky_store_fq Subroutine
procedure , private , :: prepare => ky_prepare Subroutine
procedure , private , :: make_subcom_1 => ky_make_subcom_1 Subroutine
procedure , private , :: make_subcom_2 => ky_make_subcom_2 Subroutine

type, private ::  fieldmat_type

This is the top level object, consisting of a collection of ky blocks.

Components

Type Visibility Attributes Name Initial
type(ky_type), public, dimension(:), allocatable :: kyb

The ky blocks

integer, public :: naky

Number of ky blocks

integer, public :: ntheta0
integer, public :: npts

Total number of theta grid points on all extended domains

integer, public :: nbound

Number of ignored boundary points

logical, public :: is_local

Does this supercell have any data on this proc?

logical, public :: is_empty

Have we got any data for the fieldmat on this proc?

logical, public :: is_all_local

Is all of this supercells data on this proc?

type(comm_type), public :: fm_sub_all

Sub communicator involving all processors with fieldmat

type(comm_type), public :: fm_sub_headsc_p0

Sub communicator involving the supercell heads and proc0

integer, public :: prepare_type = 0

What sort of preparation do we do to the field matrix

integer, public, dimension(:,:), allocatable :: heads

An array that holds the iproc (from comm world) of the supercell head for a given ik,it point

logical, public :: no_populate = .false.

Advanced usage only, if true then don't populate response

logical, public :: no_prepare = .false.

Advanced usage only, if true then don't prepare (invert) response

Type-Bound Procedures

procedure , private , :: deallocate => fm_deallocate Subroutine
procedure , private , :: allocate => fm_allocate Subroutine
procedure , private , :: debug_print => fm_debug_print Subroutine
procedure , private , :: get_field_update => fm_get_field_update Subroutine
procedure , private , :: init => fm_init Subroutine
procedure , private , :: reset => fm_reset Subroutine
procedure , private , :: populate => fm_populate Subroutine
procedure , private , :: init_next_field_points => fm_init_next_field_points Subroutine
procedure , private , :: prepare => fm_prepare Subroutine
procedure , private , :: make_subcom_1 => fm_make_subcom_1 Subroutine
procedure , private , :: make_subcom_2 => fm_make_subcom_2 Subroutine
procedure , private , :: calc_sc_heads => fm_calc_sc_heads Subroutine
procedure , private , :: gather_fields => fm_gather_fields Subroutine
procedure , private , :: unpack_to_field => fm_unpack_to_field Subroutine
procedure , private , :: write_debug_data => fm_write_debug_data Subroutine
procedure , private , :: set_is_local => fm_set_is_local Subroutine
procedure , private , :: count_subcom => fm_count_subcom Subroutine
procedure , private , :: getfieldeq_nogath => fm_getfieldeq_nogath Subroutine
procedure , private , :: reduce_an => fm_reduce_an_head_send_broadcast Subroutine
procedure , private , :: check_an => fm_check_an Subroutine
procedure , private , :: getfieldeq1_nogath => fm_getfieldeq1_nogath Subroutine
procedure , private , :: update_fields => fm_update_fields Subroutine
procedure , private , :: update_fields_newstep => fm_update_fields_newstep Subroutine
procedure , private , :: get_condition_numbers => fm_get_condition_numbers Subroutine

type, private ::  pc_type

This is the type which controls/organises the parallel data decomposition etc.

Components

Type Visibility Attributes Name Initial
integer, public, dimension(:,:), allocatable :: is_local

Is the given it,ik on this proc?

type(comm_type), public, dimension(:,:), allocatable :: itik_subcom

Cell level sub-communicators

integer, public, dimension(:,:), allocatable :: nproc_per_cell

How many procs have this cell

integer, public, dimension(:,:), allocatable :: nresp_per_cell

How many rows are we personally responsible for in each cell

integer, public, dimension(:,:), allocatable :: navail_per_cell

How many rows could we be responsible for in each cell

integer, public :: nresp_tot

Total number of rows available/potentially available

integer, public :: navail_tot

Total number of rows available/potentially available

logical, public :: force_local_invert = .false.

If true then we force local inversion

logical, public :: has_to_gather = .true.

If true we have to gather when calling getfieldeq, determined by decomp routine

integer, public :: decomp_type = 3

This sets what type of decomposition is done

Type-Bound Procedures

procedure , private , :: deallocate => pc_deallocate Subroutine
procedure , private , :: allocate => pc_allocate Subroutine
procedure , private , :: debug_print => pc_debug_print Subroutine
procedure , private , :: current_nresp => pc_current_nresp Function
procedure , private , :: find_locality => pc_find_locality Subroutine
procedure , private , :: count_avail => pc_count_avail Subroutine
procedure , private , :: has_ik => pc_has_ik Function
procedure , private , :: has_it => pc_has_it Function
procedure , private , :: make_decomp => pc_make_decomp Subroutine
procedure , private , :: init => pc_init Subroutine
procedure , private , :: reset => pc_reset Subroutine
procedure , private , :: decomp_all_serial_local => pc_decomp_all_serial_local Subroutine
procedure , private , :: decomp_own_serial_local => pc_decomp_own_serial_local Subroutine
procedure , private , :: decomp_owncells_serial_local => pc_decomp_owncells_serial_local Subroutine
procedure , private , :: decomp_owncells_simplempi => pc_decomp_owncells_simplempi Subroutine

Functions

private function rb_mv_mult_fun(self, vect)

Matrix vector multiplication

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(in) :: self
complex, intent(in), dimension(self%ncol) :: vect

Return Value complex, dimension(self%nrow)

private function rb_irex_to_irloc(self, irex)

Convert extended row index to local row index

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(in) :: self
integer, intent(in) :: irex

Return Value integer

private function c_has_row(self, irow)

Test if a given row belongs to the current cell

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(in) :: self
integer, intent(in) :: irow

Return Value logical

private function sc_has_it(self, it)

Is the passed it a member of this supercell

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self
integer, intent(in) :: it

Return Value logical

private function sc_get_left_it(self, it)

Given an it value get the it of the left connected cell

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self
integer, intent(in) :: it

Return Value integer

private function ky_is_from_it(self, it)

Given it say what the supercell id is

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self
integer, intent(in) :: it

Return Value integer

private function pc_current_nresp(self)

Return the current number of rows we're responsible for

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(in) :: self

Return Value integer

private function pc_has_ik(self, ik)

Determine if we have any cells with passed ik.

Read more…

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(in) :: self
integer, intent(in) :: ik

Return Value logical

private function pc_has_it(self, it)

Determine if we have any cells with passed it.

Read more…

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(in) :: self
integer, intent(in) :: it

Return Value logical

public function fields_local_functional()

Returns true if GS2 was built in such a way as to allow this module to work. Currently does not work with PGI compilers. See online discussions.

Arguments

None

Return Value logical

private function l2norm(matrix)

Calculate the L2/Frobenius/Euclidiean-norm of a 2D complex array

Arguments

Type IntentOptional Attributes Name
complex, intent(in), dimension(:, :) :: matrix

The matrix to calculate the norm of

Return Value real


Subroutines

private subroutine rb_allocate(self)

Allocate storage space

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(inout) :: self

private subroutine rb_deallocate(self)

Deallocate storage space

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(inout) :: self

private subroutine rb_init(self, ifq, ic, it, is, ik, ncol, nextend)

Initialise the members of the rowblock instance

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(inout) :: self
integer, intent(in) :: ifq
integer, intent(in) :: ic
integer, intent(in) :: it
integer, intent(in) :: is
integer, intent(in) :: ik
integer, intent(in) :: ncol
integer, intent(in) :: nextend

The length of the parent supercell's real space domain. Represents the spacing between the start of different field/field equations as we store data as the full real space domain for each field, e.g. [fieldeq(:), fieldeqa(:), fieldeqp(:)], rather than say interleaved, e.g. [[fieldeq(1), fieldeqa(1), fieldeqp(1)], [fieldeq(2), fieldeqa(2), fieldeqp(2)],...]

private subroutine rb_debug_print(self)

Debug printing

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(in) :: self

private subroutine rb_mv_mult(self, vect)

Matrix vector multiplication stored in tmp_sum

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(inout) :: self
complex, intent(in), dimension(self%ncol) :: vect

private subroutine rb_set_nrow(self)

Set the nrow variables

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(inout) :: self

private subroutine rb_reset(self)

A routine to reset the object

Arguments

Type IntentOptional Attributes Name
class(rowblock_type), intent(inout) :: self

private subroutine c_allocate(self)

Allocate storage space

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(inout) :: self

private subroutine c_deallocate(self)

Deallocate storage space

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(inout) :: self

private subroutine c_init(self, ic, it, is, ik, nfield, nrow, nextend, ignore_boundary)

Initialise the members of the cell instance

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(inout) :: self
integer, intent(in) :: ic
integer, intent(in) :: it
integer, intent(in) :: is
integer, intent(in) :: ik
integer, intent(in) :: nfield
integer, intent(in) :: nrow
integer, intent(in) :: nextend
logical, intent(in) :: ignore_boundary

private subroutine c_debug_print(self)

Debug printing

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(in) :: self

private subroutine c_mv_mult_rb(self, vect, ifq)

Do matrix vector multiplication at rowblock level

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(inout) :: self
complex, intent(in), dimension(self%ncol) :: vect
integer, intent(in) :: ifq

private subroutine c_get_field_update(self, fq, fqa, fqp)

Get the field update for this cells data Note still need to reduce across other cells in this supercell.

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(inout) :: self
complex, intent(in), dimension(self%ncol) :: fq
complex, intent(in), dimension(self%ncol) :: fqa
complex, intent(in), dimension(self%ncol) :: fqp

private subroutine c_reset(self)

A routine to reset the object

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(inout) :: self

private subroutine c_set_locality(self)

Set the locality of each object

Arguments

Type IntentOptional Attributes Name
class(cell_type), intent(inout) :: self

private subroutine sc_allocate(self)

Allocate storage space

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_deallocate(self)

Deallocate storage space

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_init(self, is, itmin, ik, nfield, nbound)

Initialise the supercell instance by setting and calculating some basic properties. Does not deal with allocating all storage etc.

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self
integer, intent(in) :: is
integer, intent(in) :: itmin
integer, intent(in) :: ik
integer, intent(in) :: nfield
integer, intent(inout) :: nbound

private subroutine sc_debug_print(self)

Debug printing

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self

private subroutine sc_get_field_update(self, fq, fqa, fqp)

Get the field update DD>TAGGED

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self
complex, intent(in), dimension(:,:) :: fq
complex, intent(in), dimension(:,:) :: fqa
complex, intent(in), dimension(:,:) :: fqp

private subroutine sc_reduce_tmpsum(self)

Reduce the field update across cells to give the final answer DD>TAGGED: As we currently have to do fm_gather_fields on every time step we only need

Read more…

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_iex_to_dims(self, iex, ig, ic, it, ifl)

Convert the extended domain index to ig, it and ifl

Read more…

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self
integer, intent(in) :: iex
integer, intent(out) :: ig
integer, intent(out) :: ic
integer, intent(out) :: it
integer, intent(out) :: ifl

private subroutine sc_iex_to_ifl(self, iex, ifl)

Convert the extended domain index to ifl

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self
integer, intent(in) :: iex
integer, intent(out) :: ifl

private subroutine sc_iex_to_ic(self, iex, ic)

Convert the extended domain index to ic

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self
integer, intent(in) :: iex
integer, intent(out) :: ic

private subroutine sc_iex_to_it(self, iex, it)

Convert the extended domain index to it

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self
integer, intent(in) :: iex
integer, intent(out) :: it

private subroutine sc_iex_to_ig(self, iex, ig)

Convert the extended domain index to ig

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self
integer, intent(in) :: iex
integer, intent(out) :: ig

private subroutine sc_reset(self)

A routine to reset the object

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_set_locality(self)

Set the locality of each object

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_dump(self, prefix)

Debug routine to dump the current supercell

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self
character(len=*), intent(in), optional :: prefix

private subroutine sc_dump_to_file(self, suffix)

Routine to write the response matrix for this supercell to netcdf file.

Read more…

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

The instance of the supercell class

character(len=*), intent(in), optional :: suffix

If passed then use as part of file suffix

private subroutine sc_read_from_file(self, could_read, suffix)

Routine to read the response matrix for this supercell from netcdf file.

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

The instance of the supercell class

logical, intent(out) :: could_read

Flag to indicate if the file was successfully read

character(len=*), intent(in), optional :: suffix

If passed then use as part of file suffix

private subroutine sc_store_fq(self, fq, fqa, fqp, ifl_in, it_in, ig_in)

Store the field equations at row level

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self
complex, intent(in), dimension(:, :) :: fq
complex, intent(in), dimension(:, :) :: fqa
complex, intent(in), dimension(:, :) :: fqp
integer, intent(in) :: ifl_in
integer, intent(in) :: it_in
integer, intent(in) :: ig_in

private subroutine sc_prepare(self, prepare_type)

Prepare the field matrix for calculating field updates

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self
integer, intent(in) :: prepare_type

private subroutine sc_invert(self)

A routine to invert the field matrix

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_invert_local(self)

A routine to invert the field matrix locally

Read more…

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_invert_mpi(self)

A routine to invert the field matrix using mpi

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_pull_rows_to_arr(self, arr)

A routine to collect all the row level data and store in passed array Gather the row blocks up for this cell to fill an array DD>FOR NOW USE ALL_REDUCE AS EASIER, BUT SHOULD BE ABLE

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(in) :: self
complex, intent(out), dimension(:,:) :: arr

private subroutine sc_push_arr_to_rows(self, arr)

A routine to distribute an array to appropriate row blocks

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self
complex, intent(in), dimension(self%nrow,self%ncol) :: arr

private subroutine sc_make_subcom_1(self)

Create primary (top level) sub communicators

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine sc_make_subcom_2(self)

Create the secondary (intraobject) subcommunicators

Arguments

Type IntentOptional Attributes Name
class(supercell_type), intent(inout) :: self

private subroutine ky_allocate(self)

Allocate storage space

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self

private subroutine ky_deallocate(self)

Deallocate storage space

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self

private subroutine ky_init(self, ik, itmins, nsupercell, nfield, nbound)

Setup properties of ky_type instance

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self
integer, intent(in) :: ik
integer, intent(in), dimension(:) :: itmins
integer, intent(in) :: nsupercell
integer, intent(in) :: nfield
integer, intent(inout) :: nbound

private subroutine ky_debug_print(self)

Debug printing

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(in) :: self

private subroutine ky_get_field_update(self, fq, fqa, fqp)

Get the field update for this ik

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self
complex, intent(in), dimension(:,:) :: fq
complex, intent(in), dimension(:,:) :: fqa
complex, intent(in), dimension(:,:) :: fqp

private subroutine ky_wait_for_nonblocking_collectives(self)

Wait on the nonblocking collective communications if they have been used.

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self

private subroutine ky_store_fq(self, fq, fqa, fqp, ifl_in, it_in, ig_in)

Store the field equation at row level

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self
complex, intent(in), dimension(:, :) :: fq
complex, intent(in), dimension(:, :) :: fqa
complex, intent(in), dimension(:, :) :: fqp
integer, intent(in) :: ifl_in
integer, intent(in) :: it_in
integer, intent(in) :: ig_in

private subroutine ky_reset(self)

A routine to reset the object

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self

private subroutine ky_set_locality(self)

Set the locality of each object

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self

private subroutine ky_prepare(self, prepare_type)

Prepare the field matrix for calculating field updates

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self
integer, intent(in) :: prepare_type

private subroutine ky_make_subcom_1(self)

Create the primary subcommunicators

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self

private subroutine ky_make_subcom_2(self)

Create the secondary subcommunicators

Arguments

Type IntentOptional Attributes Name
class(ky_type), intent(inout) :: self

private subroutine tuneMinNRow()

Arguments

None

private subroutine fm_allocate(self)

Allocate storage space

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_deallocate(self)

Deallocate storage space

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_debug_print(self)

Debug printing

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self

private subroutine fm_set_is_local(self)

Just work out the locality (also sets is_empty etc. but is not intended for this)

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_init(self)

Initialise the field objects

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_populate(self)

Find the response of g to delta-fn field perturbations and store at the row level

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_init_next_field_points(self, field, pts_remain, kwork_filter, ifl)

Initialise the next set of delta functions, find the response and store in the appropriate rows. DD>Could improve performance by using a "smart" routine which only operates on local/not empty data

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self
complex, intent(inout), dimension(-ntgrid:ntgrid,ntheta0,naky) :: field
integer, intent(inout) :: pts_remain
logical, intent(inout), dimension(ntheta0, naky) :: kwork_filter
integer, intent(in) :: ifl

private subroutine fm_prepare(self)

Prepare the field matrix for calculating field updates

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_get_field_update(self, phi, apar, bpar)

A routine to calculate the update to the fields DD>Could improve performance by using a "smart" routine which only operates on local/not empty data

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self
complex, intent(inout), dimension(:,:,:) :: phi
complex, intent(inout), dimension(:,:,:) :: apar
complex, intent(inout), dimension(:,:,:) :: bpar

private subroutine fm_unpack_to_field(self, ph, ap, bp)

A routine to unpack the supercell tmp_sum vectors to full field arrays

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self
complex, intent(inout), dimension(-ntgrid:,:,:) :: ph
complex, intent(inout), dimension(-ntgrid:,:,:) :: ap
complex, intent(inout), dimension(-ntgrid:,:,:) :: bp

private subroutine fm_reset(self)

A routine to reset the object

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_count_subcom(self)

Count how many subcommunicators will be created

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_make_subcom_1(self)

Create all the necessary subcommunicators

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_make_subcom_2(self)

Create the secondary subcommunicators

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_calc_sc_heads(self)

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self

private subroutine fm_gather_fields(self, ph, ap, bp, to_all_in, do_allreduce_in)

Gather all the fields to proc0/all for diagnostics etc. DD>TAGGED:Worth looking at improving this bad memory pattern

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(inout) :: self
complex, intent(inout), dimension(:,:,:) :: ph
complex, intent(inout), dimension(:,:,:) :: ap
complex, intent(inout), dimension(:,:,:) :: bp
logical, intent(in), optional :: to_all_in
logical, intent(in), optional :: do_allreduce_in

private subroutine fm_update_fields(self, phi, apar, bpar)

Update the fields using calculated update

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self
complex, intent(inout), dimension(:,:,:) :: phi
complex, intent(inout), dimension(:,:,:) :: apar
complex, intent(inout), dimension(:,:,:) :: bpar

private subroutine fm_update_fields_newstep(self)

Update the fields using the new fields

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self

private subroutine fm_get_condition_numbers(self)

Fetch all condition numbers to proc0 and report on worrying values as required.

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self

private subroutine fm_write_debug_data(self)

Write some debug data to file

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self

private subroutine fm_getfieldeq_nogath(self, phi, apar, bpar, fieldeq, fieldeqa, fieldeqp)

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self
complex, intent(in), dimension (-ntgrid:,:,:) :: phi
complex, intent(in), dimension (-ntgrid:,:,:) :: apar
complex, intent(in), dimension (-ntgrid:,:,:) :: bpar
complex, intent(inout), dimension (-ntgrid:,:,:) :: fieldeq
complex, intent(inout), dimension (-ntgrid:,:,:) :: fieldeqa
complex, intent(inout), dimension (-ntgrid:,:,:) :: fieldeqp

private subroutine fm_reduce_an_head_send_broadcast(self, antot, antota, antotp)

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self
complex, intent(inout), dimension (-ntgrid:,:,:) :: antot
complex, intent(inout), dimension (-ntgrid:,:,:) :: antota
complex, intent(inout), dimension (-ntgrid:,:,:) :: antotp

private subroutine fm_check_an(self, antot, tempantot, antota, tempantota, antotp, tempantotp)

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self
complex, intent(in), dimension (-ntgrid:,:,:) :: antot
complex, intent(in), dimension (-ntgrid:,:,:) :: tempantot
complex, intent(in), dimension (-ntgrid:,:,:) :: antota
complex, intent(in), dimension (-ntgrid:,:,:) :: tempantota
complex, intent(in), dimension (-ntgrid:,:,:) :: antotp
complex, intent(in), dimension (-ntgrid:,:,:) :: tempantotp

private subroutine fm_getfieldeq1_nogath(self, phi, apar, bpar, antot, antota, antotp, fieldeq, fieldeqa, fieldeqp)

Arguments

Type IntentOptional Attributes Name
class(fieldmat_type), intent(in) :: self
complex, intent(in), dimension (-ntgrid:,:,:) :: phi
complex, intent(in), dimension (-ntgrid:,:,:) :: apar
complex, intent(in), dimension (-ntgrid:,:,:) :: bpar
complex, intent(in), dimension (-ntgrid:,:,:) :: antot
complex, intent(in), dimension (-ntgrid:,:,:) :: antota
complex, intent(in), dimension (-ntgrid:,:,:) :: antotp
complex, intent(inout), dimension (-ntgrid:,:,:) :: fieldeq
complex, intent(inout), dimension (-ntgrid:,:,:) :: fieldeqa
complex, intent(inout), dimension (-ntgrid:,:,:) :: fieldeqp

private subroutine pc_allocate(self)

Allocate storage space

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_deallocate(self)

Deallocate storage space

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_debug_print(self)

Debug printing

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(in) :: self

private subroutine pc_find_locality(self)

Work out which cells are local

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_count_avail(self)

Work out how many rows are available in each cell

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_make_decomp(self)

A wrapper routine to do the decomposition

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_reset(self)

A routine to reset the pc object

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_init(self)

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_decomp_all_serial_local(self)

All serial and local || Decomp_type=0

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_decomp_own_serial_local(self)

Serial and local but only for the supercells || Decomp_type=1 for which we have some data

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_decomp_owncells_serial_local(self)

Serial and local but only for the cells || Decomp_type=2 for which we have some data

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

private subroutine pc_decomp_owncells_simplempi(self)

Cells are decomposed based on proc || Decomp_type=3

Arguments

Type IntentOptional Attributes Name
class(pc_type), intent(inout) :: self

public subroutine init_fields_local()

Routine to initialise

Arguments

None

private subroutine init_fields_matrixlocal(tuning_in)

Routine use to initialise field matrix data !!!!!!!! NOTE: All of the above should be a one off setup cost unless something fundamental changes with the parallel layout (a change in nproc/layout etc.) The remaining two tasks (populate+prepare) must be repeated each time the relevant physics and numerical parameters change. In practice this just means when the time step changes. !!!!!!!!

Arguments

Type IntentOptional Attributes Name
logical, optional :: tuning_in

public subroutine reset_fields_local()

Reset the fields_local module

Arguments

None

public subroutine finish_fields_local()

Finish the fields_local module

Arguments

None

private subroutine getfield_local(phi, apar, bpar, do_gather_in, do_update_in)

Calculate the update to the fields

Arguments

Type IntentOptional Attributes Name
complex, intent(out), dimension(:,:,:) :: phi
complex, intent(out), dimension(:,:,:) :: apar
complex, intent(out), dimension(:,:,:) :: bpar
logical, intent(in), optional :: do_gather_in
logical, intent(in), optional :: do_update_in

public subroutine init_allfields_local()

Initialise the fields from the initial g, just uses the fields_implicit routine

Arguments

None

public subroutine advance_local(istep, remove_zonal_flows_switch)

This routine advances the solution by one full time step DD>TAGGED: Should we only this is fapar>0 as well?

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: istep
logical, intent(in) :: remove_zonal_flows_switch

public subroutine dump_response_to_file_local(suffix)

Routine to dump the current response matrix data to file. One file per connected domain. Each written by the head of the supercell.

Arguments

Type IntentOptional Attributes Name
character(len=*), intent(in), optional :: suffix

If passed then use as part of file suffix

public subroutine read_response_from_file_local(could_read, suffix)

Routine to read the current response matrix data from file dump. One file per connected domain. Each written by the head of the supercell. NOTE: Have to have setup communicators etc.

Arguments

Type IntentOptional Attributes Name
logical, intent(out) :: could_read

Flag to indicate if all files successfully read

character(len=*), intent(in), optional :: suffix

If passed then use as part of file suffix