# Copyright 2016 The Gemmlowp Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """32bit ARM/NEON assembly emitter. Used by code generators to produce ARM assembly with NEON simd code. Provides tools for easier register management: named register variable allocation/deallocation, and offers a more procedural/structured approach to generating assembly. TODO: right now neon emitter prints out assembly instructions immediately, it might be beneficial to keep the whole structure and emit the assembly after applying some optimizations like: instruction reordering or register reuse. TODO: NeonRegister object assigns explicit registers at allocation time. Similarily to emiting code, register mapping and reuse can be performed and optimized lazily. """ class Error(Exception): """Module level error.""" class RegisterAllocationError(Error): """Cannot alocate registers.""" class LaneError(Error): """Wrong lane number.""" class ArgumentError(Error): """Wrong argument.""" def _Low(register): assert register[0] == 'q' num = int(register[1:]) return 'd%d' % (num * 2) def _High(register): assert register[0] == 'q' num = int(register[1:]) return 'd%d' % (num * 2 + 1) def _ExpandQuads(registers): doubles = [] for register in registers: if register[0] == 'q': doubles.append(_Low(register)) doubles.append(_High(register)) else: doubles.append(register) return doubles def _MakeCompatible(op1, op2, op3): if op1[0] == 'd' or op2[0] == 'd' or op3[0] == 'd': if op1[0] == 'q': op1 = _Low(op1) if op2[0] == 'q': op2 = _Low(op2) if op3[0] == 'q': op3 = _Low(op3) return (op1, op2, op3) class _NeonRegisters32Bit(object): """Utility that keeps track of used 32bit ARM/NEON registers.""" def __init__(self): self.double = set() self.double_ever = set() self.general = set() self.general_ever = set() self.parameters = dict() self.output_parameters = dict() def MapParameter(self, parameter, parameter_value=None): if not parameter_value: parameter_value = parameter self.parameters[parameter] = (parameter_value, 'r') return '%%[%s]' % parameter def MapMemoryParameter(self, parameter, parameter_value=None): if not parameter_value: parameter_value = parameter self.parameters[parameter] = (parameter_value, 'm') return '%%[%s]' % parameter def MapOutputParameter(self, parameter, parameter_value=None): if not parameter_value: parameter_value = parameter self.output_parameters[parameter] = (parameter_value, '+r') return '%%[%s]' % parameter def DoubleRegister(self, min_val=0): for i in range(min_val, 32): if i not in self.double: self.double.add(i) self.double_ever.add(i) return 'd%d' % i raise RegisterAllocationError('Not enough double registers.') def QuadRegister(self, min_val=0): for i in range(min_val, 16): if ((i * 2) not in self.double) and ((i * 2 + 1) not in self.double): self.double.add(i * 2) self.double.add(i * 2 + 1) self.double_ever.add(i * 2) self.double_ever.add(i * 2 + 1) return 'q%d' % i raise RegisterAllocationError('Not enough quad registers.') def GeneralRegister(self): for i in range(0, 16): if i not in self.general: self.general.add(i) self.general_ever.add(i) return 'r%d' % i raise RegisterAllocationError('Not enough general registers.') def MappedParameters(self): return [(k, v) for (k, v) in self.parameters.items()] def MappedOutputParameters(self): return [(k, v) for (k, v) in self.output_parameters.items()] def Clobbers(self): return (['r%d' % i for i in self.general_ever] + ['d%d' % i for i in self.DoubleClobbers()]) def DoubleClobbers(self): return sorted(self.double_ever) def FreeRegister(self, register): assert len(register) > 1 if register[0] not in ['r', 'd', 'q']: return num = int(register[1:]) if register[0] == 'r': assert num in self.general self.general.remove(num) elif register[0] == 'd': assert num in self.double self.double.remove(num) elif register[0] == 'q': assert num * 2 in self.double assert num * 2 + 1 in self.double self.double.remove(num * 2) self.double.remove(num * 2 + 1) else: raise RegisterDeallocationError('Register not allocated: %s' % register) def FreeRegisters(self, registers): for register in registers: self.FreeRegister(register) class NeonEmitter(object): """Emits ARM/NEON assembly opcodes.""" def __init__(self, debug=False): self.ops = {} self.indent = '' self.debug = debug def PushIndent(self, delta=' '): self.indent += delta def PopIndent(self, delta=2): self.indent = self.indent[:-delta] def EmitIndented(self, what): print(self.indent + what) def PushOp(self, op): if op in self.ops.keys(): self.ops[op] += 1 else: self.ops[op] = 1 def ClearCounters(self): self.ops.clear() def EmitNewline(self): print('') def EmitPreprocessor1(self, op, param): print('#%s %s' % (op, param)) def EmitPreprocessor(self, op): print('#%s' % op) def EmitInclude(self, include): self.EmitPreprocessor1('include', include) def EmitCall1(self, function, param): self.EmitIndented('%s(%s);' % (function, param)) def EmitAssert(self, assert_expression): if self.debug: self.EmitCall1('assert', assert_expression) def EmitHeaderBegin(self, header_name, includes): self.EmitPreprocessor1('ifndef', (header_name + '_H_').upper()) self.EmitPreprocessor1('define', (header_name + '_H_').upper()) self.EmitNewline() if includes: for include in includes: self.EmitInclude(include) self.EmitNewline() def EmitHeaderEnd(self): self.EmitPreprocessor('endif') def EmitCode(self, code): self.EmitIndented('%s;' % code) def EmitFunctionBeginA(self, function_name, params, return_type): self.EmitIndented('%s %s(%s) {' % (return_type, function_name, ', '.join(['%s %s' % (t, n) for (t, n) in params]))) self.PushIndent() def EmitFunctionEnd(self): self.PopIndent() self.EmitIndented('}') def EmitAsmBegin(self): self.EmitIndented('asm volatile(') self.PushIndent() def EmitAsmMapping(self, elements): if elements: self.EmitIndented(': ' + ', '.join( ['[%s] "%s"(%s)' % (d, v[1], v[0]) for (d, v) in elements])) else: self.EmitIndented(':') def EmitClobbers(self, elements): if elements: self.EmitIndented(': ' + ', '.join(['"%s"' % c for c in elements])) else: self.EmitIndented(':') def EmitAsmEnd(self, registers): self.EmitAsmMapping(registers.MappedOutputParameters()) self.EmitAsmMapping(registers.MappedParameters()) self.EmitClobbers(registers.Clobbers() + ['cc', 'memory']) self.PopIndent() self.EmitIndented(');') def EmitComment(self, comment): self.EmitIndented('// ' + comment) def EmitNumericalLabel(self, label): self.EmitIndented('"%d:"' % label) def EmitOp1(self, op, param1): self.PushOp(op) self.EmitIndented('"%s %s\\n"' % (op, param1)) def EmitOp2(self, op, param1, param2): self.PushOp(op) self.EmitIndented('"%s %s, %s\\n"' % (op, param1, param2)) def EmitOp3(self, op, param1, param2, param3): self.PushOp(op) self.EmitIndented('"%s %s, %s, %s\\n"' % (op, param1, param2, param3)) def EmitAdd(self, destination, source, param): self.EmitOp3('add', destination, source, param) def EmitSubs(self, destination, source, param): self.EmitOp3('subs', destination, source, param) def EmitSub(self, destination, source, param): self.EmitOp3('sub', destination, source, param) def EmitMul(self, destination, source, param): self.EmitOp3('mul', destination, source, param) def EmitMov(self, param1, param2): self.EmitOp2('mov', param1, param2) def EmitBeqBack(self, label): self.EmitOp1('beq', '%db' % label) def EmitBeqFront(self, label): self.EmitOp1('beq', '%df' % label) def EmitBgtBack(self, label): self.EmitOp1('bgt', '%db' % label) def EmitBgtFront(self, label): self.EmitOp1('bgt', '%df' % label) def EmitBleBack(self, label): self.EmitOp1('ble', '%db' % label) def EmitBleFront(self, label): self.EmitOp1('ble', '%df' % label) def EmitBneBack(self, label): self.EmitOp1('bne', '%db' % label) def EmitBneFront(self, label): self.EmitOp1('bne', '%df' % label) def EmitVAdd(self, add_type, destination, source_1, source_2): destination, source_1, source_2 = _MakeCompatible(destination, source_1, source_2) self.EmitOp3('vadd.%s' % add_type, destination, source_1, source_2) def EmitVAddw(self, add_type, destination, source_1, source_2): self.EmitOp3('vaddw.%s' % add_type, destination, source_1, source_2) def EmitVSub(self, sub_type, destination, source_1, source_2): destination, source_1, source_2 = _MakeCompatible(destination, source_1, source_2) self.EmitOp3('vsub.%s' % sub_type, destination, source_1, source_2) def EmitVCvt(self, cvt_to, cvt_from, destination, source): self.EmitOp2('vcvt.%s.%s' % (cvt_to, cvt_from), destination, source) def EmitVDup(self, dup_type, destination, source): self.EmitOp2('vdup.%s' % dup_type, destination, source) def EmitVMax(self, size, destination, source_1, source_2): self.EmitOp3('vmax.%s' % size, destination, source_1, source_2) def EmitVMin(self, size, destination, source_1, source_2): self.EmitOp3('vmin.%s' % size, destination, source_1, source_2) def EmitVMov(self, mov_type, destination, source): self.EmitOp2('vmov.%s' % mov_type, destination, source) def EmitVMovl(self, mov_type, destination, source): if source[0] == 'q': source = _Low(source) self.EmitOp2('vmovl.%s' % mov_type, destination, source) def EmitVMovl2(self, mov_type, destination_1, destination_2, source): self.EmitVMovl(mov_type, destination_2, _High(source)) self.EmitVMovl(mov_type, destination_1, _Low(source)) def EmitVQmovn(self, mov_type, destination, source): if destination[0] == 'q': destination = _Low(destination) self.EmitOp2('vqmovn.%s' % mov_type, destination, source) def EmitVQmovn2(self, mov_type, destination, source_1, source_2): self.EmitVQmovn(mov_type, _Low(destination), source_1) self.EmitVQmovn(mov_type, _High(destination), source_2) def EmitVQmovun(self, mov_type, destination, source): if destination[0] == 'q': destination = _Low(destination) self.EmitOp2('vqmovun.%s' % mov_type, destination, source) def EmitVQmovun2(self, mov_type, destination, source_1, source_2): self.EmitVQmovun(mov_type, _Low(destination), source_1) self.EmitVQmovun(mov_type, _High(destination), source_2) def EmitVMul(self, mul_type, destination, source_1, source_2): destination, source_1, source_2 = _MakeCompatible(destination, source_1, source_2) self.EmitOp3('vmul.%s' % mul_type, destination, source_1, source_2) def EmitVMulScalar(self, mul_type, destination, source_1, source_2): self.EmitOp3('vmul.%s' % mul_type, destination, source_1, source_2) def EmitVMull(self, mul_type, destination, source_1, source_2): self.EmitOp3('vmull.%s' % mul_type, destination, source_1, source_2) def EmitVPadd(self, add_type, destination, source_1, source_2): self.EmitOp3('vpadd.%s' % add_type, destination, source_1, source_2) def EmitVPaddl(self, add_type, destination, source): self.EmitOp2('vpaddl.%s' % add_type, destination, source) def EmitVPadal(self, add_type, destination, source): self.EmitOp2('vpadal.%s' % add_type, destination, source) def EmitLdr(self, register, value): self.EmitOp2('ldr', register, value) def EmitVLoad(self, load_no, load_type, destination, source): self.EmitVLoadA(load_no, load_type, [destination], source) def EmitVLoadA(self, load_no, load_type, destinations, source): self.EmitOp2('vld%d.%d' % (load_no, load_type), '{%s}' % ', '.join(_ExpandQuads(destinations)), source) def EmitVLoadAE(self, load_type, elem_count, destinations, source, alignment=None): bits_to_load = load_type * elem_count destinations = _ExpandQuads(destinations) if len(destinations) * 64 < bits_to_load: raise ArgumentError('To few destinations: %d to load %d bits.' % (len(destinations), bits_to_load)) while bits_to_load > 0: if bits_to_load >= 256: self.EmitVLoadA(1, 32, destinations[:4], self.DereferenceIncrement(source, alignment)) bits_to_load -= 256 destinations = destinations[4:] elif bits_to_load >= 192: self.EmitVLoadA(1, 32, destinations[:3], self.DereferenceIncrement(source, alignment)) bits_to_load -= 192 destinations = destinations[3:] elif bits_to_load >= 128: self.EmitVLoadA(1, 32, destinations[:2], self.DereferenceIncrement(source, alignment)) bits_to_load -= 128 destinations = destinations[2:] elif bits_to_load >= 64: self.EmitVLoad(1, 32, destinations[0], self.DereferenceIncrement(source, alignment)) bits_to_load -= 64 destinations = destinations[1:] else: destination = destinations[0] if bits_to_load == 56: self.EmitVLoad(1, 32, self.Lane(32, destination, 0), self.DereferenceIncrement(source)) self.EmitVLoad(1, 16, self.Lane(16, destination, 2), self.DereferenceIncrement(source)) self.EmitVLoad(1, 8, self.Lane(8, destination, 6), self.DereferenceIncrement(source)) elif bits_to_load == 48: self.EmitVLoad(1, 32, self.Lane(32, destination, 0), self.DereferenceIncrement(source)) self.EmitVLoad(1, 16, self.Lane(16, destination, 2), self.DereferenceIncrement(source)) elif bits_to_load == 40: self.EmitVLoad(1, 32, self.Lane(32, destination, 0), self.DereferenceIncrement(source)) self.EmitVLoad(1, 8, self.Lane(8, destination, 4), self.DereferenceIncrement(source)) elif bits_to_load == 32: self.EmitVLoad(1, 32, self.Lane(32, destination, 0), self.DereferenceIncrement(source)) elif bits_to_load == 24: self.EmitVLoad(1, 16, self.Lane(16, destination, 0), self.DereferenceIncrement(source)) self.EmitVLoad(1, 8, self.Lane(8, destination, 2), self.DereferenceIncrement(source)) elif bits_to_load == 16: self.EmitVLoad(1, 16, self.Lane(16, destination, 0), self.DereferenceIncrement(source)) elif bits_to_load == 8: self.EmitVLoad(1, 8, self.Lane(8, destination, 0), self.DereferenceIncrement(source)) else: raise ArgumentError('Wrong leftover: %d' % bits_to_load) return def EmitVLoadE(self, load_type, count, destination, source, alignment=None): self.EmitVLoadAE(load_type, count, [destination], source, alignment) def EmitVLoadAllLanes(self, load_no, load_type, destination, source): destinations = [] if destination[0] == 'q': destinations.append(self.AllLanes(_Low(destination))) destinations.append(self.AllLanes(_High(destination))) else: destinations.append(self.AllLanes(destination)) self.EmitVLoadA(load_no, load_type, destinations, source) def EmitVLoadOffset(self, load_no, load_type, destination, source, offset): self.EmitVLoadOffsetA(load_no, load_type, [destination], source, offset) def EmitVLoadOffsetA(self, load_no, load_type, destinations, source, offset): assert len(destinations) <= 4 self.EmitOp3('vld%d.%d' % (load_no, load_type), '{%s}' % ', '.join(_ExpandQuads(destinations)), source, offset) def EmitPld(self, load_address_register): self.EmitOp1('pld', '[%s]' % load_address_register) def EmitPldw(self, store_address_register): self.EmitOp1('pldw', '[%s]' % store_address_register) def EmitPldOffset(self, load_address_register, offset): self.EmitOp1('pld', '[%s, %s]' % (load_address_register, offset)) def EmitPldwOffset(self, store_address_register, offset): self.EmitOp1('pldw', '[%s, %s]' % (store_address_register, offset)) def EmitVShl(self, shift_type, destination, source, shift): self.EmitOp3('vshl.%s' % shift_type, destination, source, shift) def EmitVStore(self, store_no, store_type, source, destination): self.EmitVStoreA(store_no, store_type, [source], destination) def EmitVStoreA(self, store_no, store_type, sources, destination): self.EmitOp2('vst%d.%d' % (store_no, store_type), '{%s}' % ', '.join(_ExpandQuads(sources)), destination) def EmitVStoreAE(self, store_type, elem_count, sources, destination, alignment=None): bits_to_store = store_type * elem_count sources = _ExpandQuads(sources) if len(sources) * 64 < bits_to_store: raise ArgumentError('To few sources: %d to store %d bits.' % (len(sources), bits_to_store)) while bits_to_store > 0: if bits_to_store >= 256: self.EmitVStoreA(1, 32, sources[:4], self.DereferenceIncrement(destination, alignment)) bits_to_store -= 256 sources = sources[4:] elif bits_to_store >= 192: self.EmitVStoreA(1, 32, sources[:3], self.DereferenceIncrement(destination, alignment)) bits_to_store -= 192 sources = sources[3:] elif bits_to_store >= 128: self.EmitVStoreA(1, 32, sources[:2], self.DereferenceIncrement(destination, alignment)) bits_to_store -= 128 sources = sources[2:] elif bits_to_store >= 64: self.EmitVStore(1, 32, sources[0], self.DereferenceIncrement(destination, alignment)) bits_to_store -= 64 sources = sources[1:] else: source = sources[0] if bits_to_store == 56: self.EmitVStore(1, 32, self.Lane(32, source, 0), self.DereferenceIncrement(destination)) self.EmitVStore(1, 16, self.Lane(16, source, 2), self.DereferenceIncrement(destination)) self.EmitVStore(1, 8, self.Lane(8, source, 6), self.DereferenceIncrement(destination)) elif bits_to_store == 48: self.EmitVStore(1, 32, self.Lane(32, source, 0), self.DereferenceIncrement(destination)) self.EmitVStore(1, 16, self.Lane(16, source, 2), self.DereferenceIncrement(destination)) elif bits_to_store == 40: self.EmitVStore(1, 32, self.Lane(32, source, 0), self.DereferenceIncrement(destination)) self.EmitVStore(1, 8, self.Lane(8, source, 4), self.DereferenceIncrement(destination)) elif bits_to_store == 32: self.EmitVStore(1, 32, self.Lane(32, source, 0), self.DereferenceIncrement(destination)) elif bits_to_store == 24: self.EmitVStore(1, 16, self.Lane(16, source, 0), self.DereferenceIncrement(destination)) self.EmitVStore(1, 8, self.Lane(8, source, 2), self.DereferenceIncrement(destination)) elif bits_to_store == 16: self.EmitVStore(1, 16, self.Lane(16, source, 0), self.DereferenceIncrement(destination)) elif bits_to_store == 8: self.EmitVStore(1, 8, self.Lane(8, source, 0), self.DereferenceIncrement(destination)) else: raise ArgumentError('Wrong leftover: %d' % bits_to_store) return def EmitVStoreE(self, store_type, count, source, destination, alignment=None): self.EmitVStoreAE(store_type, count, [source], destination, alignment) def EmitVStoreOffset(self, store_no, store_type, source, destination, offset): self.EmitVStoreOffsetA(store_no, store_type, [source], destination, offset) def EmitVStoreOffsetA(self, store_no, store_type, sources, destination, offset): self.EmitOp3('vst%d.%d' % (store_no, store_type), '{%s}' % ', '.join(_ExpandQuads(sources)), destination, offset) def EmitVStoreOffsetE(self, store_type, count, source, destination, offset): """Emit assembly to store a number elements from the source registers.""" if store_type is not 32: raise ArgumentError('Unsupported store_type: %d' % store_type) sources = [] if source[0] == 'q': sources.append(_Low(source)) sources.append(_High(source)) if count * store_type > 128: raise ArgumentError('To many %dbit elements in a q register: %d' % (store_type, count)) else: sources.append(source) if count * store_type > 64: raise ArgumentError('To many %dbit elements in a d register: %d' % (store_type, count)) if count == 1: self.EmitVStoreOffset(1, store_type, self.Lane(store_type, sources[0], 0), self.Dereference(destination, None), offset) elif count == 2: self.EmitVStoreOffset(1, store_type, sources[0], self.Dereference(destination, None), offset) elif count == 3: self.EmitVStore(1, store_type, sources[0], self.DereferenceIncrement(destination, None)) self.EmitVStoreOffset(1, store_type, self.Lane(store_type, sources[1], 0), self.Dereference(destination, None), offset) self.EmitSub(destination, destination, self.ImmediateConstant(8)) elif count == 4: self.EmitVStoreOffsetA(1, store_type, sources, self.Dereference(destination, None), offset) else: raise ArgumentError('To many elements: %d' % count) def EmitVSumReduce(self, reduce_type, elem_count, reduce_count, destinations, sources): """Emit assembly for n-fold horizontal sum reduction.""" if reduce_type is not 'u32': raise ArgumentError('Unsupported reduce: %s' % reduce_type) sources = _ExpandQuads(sources) destinations = _ExpandQuads(destinations) if len(destinations) * 2 < elem_count: raise ArgumentError('Not enough space in destination: %d vs %d' % (len(destinations) * 2, elem_count)) if len(sources) * 2 != elem_count * reduce_count: raise ArgumentError('Wrong number of sources: %d vs %d' % (len(sources) * 2, elem_count * reduce_count)) if reduce_count <= 1: raise ArgumentError('Unsupported reduce_count: %d' % reduce_count) while reduce_count > 1: if len(sources) % 2 == 1: sources.append(sources[-1]) if reduce_count == 2: for i in range(len(sources) / 2): self.EmitVPadd(reduce_type, destinations[i], sources[2 * i], sources[2 * i + 1]) return else: sources_2 = [] for i in range(len(sources) / 2): self.EmitVPadd(reduce_type, sources[2 * i], sources[2 * i], sources[2 * i + 1]) sources_2.append(sources[2 * i]) reduce_count /= 2 sources = sources_2 def EmitVUzp(self, uzp_type, operand_1, operand_2): self.EmitOp2('vuzp.%d' % uzp_type, operand_1, operand_2) def EmitVTrn(self, trn_type, operand_1, operand_2): self.EmitOp2('vtrn.%d' % trn_type, operand_1, operand_2) def EmitColBlockStride(self, cols, stride, new_stride): assert cols in [1, 2, 3, 4, 5, 6, 7, 8] if cols in [5, 6, 7]: self.EmitSub(new_stride, stride, self.ImmediateConstant(4)) def EmitLoadColBlock(self, unused_registers, load_type, cols, elements, block, input_address, stride): """Load a block of column major data.""" assert cols is len(block) assert load_type is 8 input_deref = self.Dereference(input_address, None) input_deref_increment = self.DereferenceIncrement(input_address, None) if cols is 1: for i in range(elements): self.EmitVLoadOffset(1, 8, self.Lane(8, block[0], i), input_deref, stride) self.EmitPld(input_address) elif cols is 2: for i in range(elements): self.EmitVLoadOffset(1, 16, self.Lane(16, block[i / 4], i % 4), input_deref, stride) self.EmitPld(input_address) self.EmitVUzp(8, block[0], block[1]) elif cols is 3: for i in range(elements): self.EmitVLoadOffsetA(3, 8, [self.Lane(8, row, i) for row in block], input_deref, stride) elif cols is 4: for i in range(elements): self.EmitVLoadOffset(1, 32, self.Lane(32, block[i % 4], i / 4), input_deref, stride) self.EmitPld(input_address) self.EmitVTrn(16, block[0], block[2]) self.EmitVTrn(16, block[1], block[3]) self.EmitVTrn(8, block[0], block[1]) self.EmitVTrn(8, block[2], block[3]) elif cols is 5: for i in range(elements): self.EmitVLoad(1, 32, self.Lane(32, block[i % 4], i / 4), input_deref_increment) self.EmitVLoadOffset(1, 8, self.Lane(8, block[4], i), input_deref, stride) self.EmitPld(input_address) self.EmitVTrn(16, block[0], block[2]) self.EmitVTrn(16, block[1], block[3]) self.EmitVTrn(8, block[0], block[1]) self.EmitVTrn(8, block[2], block[3]) elif cols is 6: for i in range(elements): self.EmitVLoad(1, 32, self.Lane(32, block[i % 4], i / 4), input_deref_increment) self.EmitVLoadOffset(1, 16, self.Lane(16, block[4 + i / 4], i % 4), input_deref, stride) self.EmitPld(input_address) self.EmitVTrn(16, block[0], block[2]) self.EmitVTrn(16, block[1], block[3]) self.EmitVUzp(8, block[4], block[5]) self.EmitVTrn(8, block[0], block[1]) self.EmitVTrn(8, block[2], block[3]) elif cols is 7: for i in range(elements): self.EmitVLoad(1, 32, self.Lane(32, block[i % 4], i / 4), input_deref_increment) self.EmitVLoadOffsetA(3, 8, [self.Lane(8, row, i) for row in block[4:]], input_deref, stride) self.EmitPld(input_address) self.EmitVTrn(16, block[0], block[2]) self.EmitVTrn(16, block[1], block[3]) self.EmitVTrn(8, block[0], block[1]) self.EmitVTrn(8, block[2], block[3]) elif cols is 8: for i in range(elements): self.EmitVLoadOffset(1, 32, block[i], input_deref, stride) self.EmitPld(input_address) self.EmitVTrn(8, block[0], block[1]) self.EmitVTrn(8, block[2], block[3]) self.EmitVTrn(8, block[4], block[5]) self.EmitVTrn(8, block[6], block[7]) self.EmitVTrn(16, block[0], block[2]) self.EmitVTrn(16, block[1], block[3]) self.EmitVTrn(16, block[4], block[6]) self.EmitVTrn(16, block[5], block[7]) self.EmitVTrn(32, block[0], block[4]) self.EmitVTrn(32, block[1], block[5]) self.EmitVTrn(32, block[2], block[6]) self.EmitVTrn(32, block[3], block[7]) else: assert False return block def Dereference(self, value, alignment=None): if alignment: return '[%s:%d]' % (value, alignment) else: return '[%s]' % value def DereferenceIncrement(self, value, alignment=None): return '%s!' % self.Dereference(value, alignment) def ImmediateConstant(self, value): return '#%d' % value def AllLanes(self, value): return '%s[]' % value def Lane(self, bits, value, lane): """Get the proper n-bit lane from the given register.""" registers = [] if value[0] == 'q': registers.append(_Low(value)) registers.append(_High(value)) else: registers.append(value) elems_per_register = 64 / bits register = lane / elems_per_register lane %= elems_per_register return '%s[%d]' % (registers[register], lane) def CreateRegisters(self): return _NeonRegisters32Bit()