Low accuracy Any help will be appreciated

Hi everyone , i am receiving a very low accuracy of around 0.4 . I am attaching my code below . I have tried everything changing learning rate , different layers initlizers, augmentation, MSE loss, but this network does not learn anything.0.4 accuracy is because it keep predicting only class 2 and class 4 in each prediction, Kindly any tip will be appreciated.
I also have tried cropping different cubes from the cube trained them but that method still didn’t work. Code is attached below.

from segmentation_models import Unet, FPN

base_model = Unet(backbone_name=‘resnet34’, encoder_weights=None, input_shape=(256, 224, 1))
base_model.layers.pop()
x1 = base_model.output
x1 = Conv2D(6, 5 ,strides=1, padding=‘same’, activation=‘relu’)(x1)
out = Conv2D(6, kernel_size =5,strides=1,padding = “same”)(x1)
model = Model(base_model.input, out)

x= data_train.reshape(1006,782,590,1)
y = labels_train.reshape(1006,782,590,1).astype(‘int’)-1

def reshape_images(inputs, labels=None, shapess=(224,256)):
tmp = np.zeros((inputs.shape[0], 256,224))
for i,each_input in enumerate(inputs):
tmp[i] = cv2.resize(each_input, dsize=shapess)
if labels is not None:
tmp1 = np.zeros((labels.shape[0], 256,224))
for i,each_input in enumerate(labels):
tmp1[i] = cv2.resize(each_input.astype(‘float’), dsize=shapess)
return tmp,tmp1

return tmp

X,Y = reshape_images(x,y)
Y= Y.astype(‘int’)
X = X.reshape(-1,256,224,1)
Y = Y.reshape(-1,256,224,1).astype(‘int’)

data_gen_args = dict(
rotation_range=90.,
width_shift_range=0.5,
height_shift_range=0.5,
zoom_range=0.5, validation_split=0.25)
image_datagen = ImageDataGenerator(**data_gen_args)
mask_datagen = ImageDataGenerator(**data_gen_args)

seed = 1
image_datagen.fit(X, augment=True, seed=seed )
mask_datagen.fit(Y, augment=True, seed=seed)

image_generator = image_datagen.flow(X,seed=seed, subset=‘training’, batch_size=2)

mask_generator = mask_datagen.flow(Y.astype(‘int’),seed=seed, subset=‘training’, batch_size=2)
train_generator = zip(image_generator, mask_generator)

val_image_generator = image_datagen.flow(X,seed=seed, subset=‘validation’, batch_size=2)

val_mask_generator = mask_datagen.flow(Y.astype(‘int’),seed=seed, subset=‘validation’, batch_size=2)
val_train_generator = zip(val_image_generator, val_mask_generator)

optimizer = keras.optimizers.SGD(learning_rate=0.00001, clipvalue=0.5)
model.compile(optimizer=optimizer , loss= ‘sparse_categorical_crossentropy’, metrics=[‘accuracy’])
model.fit_generator(train_generator, validation_data=val_train_generator, steps_per_epoch=50, epochs =500, validation_steps=10)